{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "# -*- coding: utf-8 -*-\n",
    "import numpy as np\n",
    "import torch\n",
    "from matplotlib import pylab as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def running_mean(x, N=50):\n",
    "    cumsum = np.cumsum(np.insert(x, 0, 0)) \n",
    "    return (cumsum[N:] - cumsum[:-N]) / float(N)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "l1 = 4\n",
    "l2 = 150\n",
    "l3 = 2\n",
    "\n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(l1, l2),\n",
    "    torch.nn.LeakyReLU(),\n",
    "    torch.nn.Linear(l2, l3),\n",
    "    torch.nn.Softmax()\n",
    ")\n",
    "\n",
    "#loss_fn = torch.nn.MSELoss(size_average=False)\n",
    "def loss_fn_old(pred, a, r, d): \n",
    "    #pred is output from neural network, a is action index\n",
    "    #r is return (sum of rewards to end of episode), d is discount factor\n",
    "    return -1 * r * d * torch.log(pred[a]) #element-wise multipliy, then sum\n",
    "\n",
    "def loss_fn(preds, r): \n",
    "    #pred is output from neural network, a is action index\n",
    "    #r is return (sum of rewards to end of episode), d is discount factor\n",
    "    return -1 * torch.sum(r * torch.log(preds)) #element-wise multipliy, then sum\n",
    "\n",
    "learning_rate = 0.0009\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "observation = env.reset()\n",
    "#state, reward, done, info = env.step(1)\n",
    "#observation.shape # (4,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([ 0.5085,  0.4915])\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/brandonbrown/anaconda3/envs/deeprl/lib/python3.6/site-packages/torch/nn/modules/container.py:91: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  input = module(input)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor(63.9182)"
      ]
     },
     "execution_count": 195,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred = model(Variable(torch.from_numpy(observation).float()))\n",
    "print(pred) #type: float tensor\n",
    "a = int(np.random.choice(np.array([0,1]), p=pred.data.numpy())) #action\n",
    "r = 100 #reward\n",
    "d = 0.9 #discount\n",
    "loss_fn(pred, a, r, d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def discount_rewards(rewards, gamma=0.99): #rewards is a sequence e.g. [50,49,48,47,...]\n",
    "    lenr = len(rewards)\n",
    "    d_rewards = torch.pow(gamma,torch.arange(lenr)) * rewards #discounted rewards\n",
    "    d_rewards = (d_rewards - d_rewards.mean()) / (d_rewards.std() + 1e-07)\n",
    "    return d_rewards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 1.9257,  1.8240,  1.7240,  1.6256,  1.5289,  1.4338,  1.3404,\n",
       "         1.2484,  1.1581,  1.0693,  0.9820,  0.8961,  0.8118,  0.7289,\n",
       "         0.6474,  0.5674,  0.4887,  0.4114,  0.3354,  0.2608,  0.1875,\n",
       "         0.1154,  0.0447, -0.0248, -0.0931, -0.1601, -0.2260, -0.2906,\n",
       "        -0.3541, -0.4165, -0.4777, -0.5378, -0.5968, -0.6547, -0.7115,\n",
       "        -0.7673, -0.8220, -0.8758, -0.9285, -0.9802, -1.0309, -1.0807,\n",
       "        -1.1295, -1.1774, -1.2244, -1.2705, -1.3156, -1.3599, -1.4033,\n",
       "        -1.4459])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "discount_rewards(torch.arange(50, 0, -1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/brandonbrown/anaconda3/envs/deeprl/lib/python3.6/site-packages/torch/nn/modules/container.py:91: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  input = module(input)\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "MAX_DUR = 200\n",
    "MAX_EPISODES = 500\n",
    "gamma_ = 0.99\n",
    "losses = []\n",
    "time_steps = []\n",
    "for episode in range(MAX_EPISODES):\n",
    "    state1 = env.reset()\n",
    "    done = False\n",
    "    t = 0\n",
    "    obs = [] # list of state observations\n",
    "    actions = [] #list of actions\n",
    "    while not done: #while in episode\n",
    "        #env.render()\n",
    "        pred = model(torch.from_numpy(state1).float())\n",
    "        action = np.random.choice(np.array([0,1]), p=pred.data.numpy())\n",
    "        state2, reward, done, info = env.step(action)\n",
    "        obs.append(state1)\n",
    "        actions.append(action)\n",
    "        state1 = state2\n",
    "        t += 1\n",
    "        if t > MAX_DUR:\n",
    "            break;\n",
    "    time_steps.append(t+1)\n",
    "    #print(\"Episode finished after {} timesteps\".format(t+1))\n",
    "    # Optimize policy network with full episode\n",
    "    ep_len = len(obs) # episode length\n",
    "    rewards = torch.arange(ep_len, 0, -1) #list of rewards\n",
    "    preds = torch.zeros(ep_len)\n",
    "    for j in range(ep_len): #for each step in episode\n",
    "        d_rewards = discount_rewards(rewards, gamma_)\n",
    "        state = obs[j]\n",
    "        action = int(actions[j])\n",
    "        pred = model(torch.from_numpy(state).float())\n",
    "        preds[j] = pred[action]\n",
    "        \n",
    "    loss = loss_fn(preds, d_rewards)\n",
    "    losses.append(loss)\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "        \n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmcAAAGtCAYAAABTKdNeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xd4FVXixvHvSQ8ltISWEHqRIhBCUYogfVEBQbFSFQvrT10L9tVVV9e+NhQRBVSKCIIovYsIJEgx1BBaSJAQAgQSSDu/PxKzoAECcjM3ue/neXi4OTNz73t5FvflzJwZY61FRERERNyDl9MBREREROR/VM5ERERE3IjKmYiIiIgbUTkTERERcSMqZyIiIiJuROVMRERExI24rJwZY8YbYw4ZY349Y6yFMeZnY8wGY0yUMaZN3rgxxrxrjIk1xmwyxkS4KpeIiIiIO3PlzNnnQK8/jL0GvGCtbQE8l/czQG+gft6vkcAYF+YSERERcVsuK2fW2hXAkT8OA0F5r8sBCXmv+wITba6fgfLGmGquyiYiIiLirnyK+PMeAuYbY94gtxhenTceCuw/Y7/4vLHE871ZcHCwrVWrlgtiioiIiFxe0dHRh621IRfar6jL2X3Aw9bab4wxNwOfAt0AU8C+BT5XyhgzktxTn4SHhxMVFeWqrCIiIiKXjTFmb2H2K+rVmkOAGXmvvwba5L2OB2qcsV8Y/zvleRZr7VhrbaS1NjIk5ILlU0RERKRYKepylgBck/f6WmBn3uvZwOC8VZvtgGPW2vOe0hQREREpiVx2WtMYMxnoDAQbY+KBfwJ3A/81xvgAp8g7PQn8APwNiAXSgGGuyiUiIiLizlxWzqy1t55jU6sC9rXAKFdlERERESku9IQAERERETeiciYiIiLiRlTORERERNyIypmIiIiIG1E5ExEREXEjKmciIiIibkTlTERERMSNqJyJiIiIuBGVMxERERE3onImIiIi4kZUzkRERMSj7T26l33H9jkdI5/KmYiIiHiM3Md5/+/1Q/Meot579Xh26bMOpjqbypmIiIgUC1k5WayJX3NWwSqs46eP039qf5qNaUaOzQFg+d7l/HfNf7m16a28fO3LlzvuJVM5ExEREbdjreWOGXfw6o+v5o89segJ2n3ajhlbZ/xp/8TURBbuWkh0QjRDvx3KkfQjZ20fNmsY3277lpikGObsmMMdM+7gheUvUCGgAh9f9zFhQWEu/06F5eN0ABERESmZvt32LbuO7OKRqx+56GNnbpvJl5u/pLRvae6NvJfYI7G8/fPbGAzPLH2Gfo364e3lnb//6EWjmbRpEj5ePmTlZBFSKoTjp49zMvMkver1YubWmQxrMYzPNnzG8FnDSU5PBuDBtg8S6Bt42b7z5WAuZWrQXURGRtqoqCinY4iIiMgZ4o/Hs/HgRgZ+PZDM7EySHkuiQmCFQh9vraXh+w1Jy0zjQOoBHr/6cWZum8nJzJO82OVFRsweweQBk7ml6S35+4e/E46ftx81y9Uk22azYu8KvIwXFQMrcjjtMN7Gmz0P7aHnFz3ZkrSFLrW6cGWVK3m8/eNUL1vdVX8UZzHGRFtrIy+0n05rioiIyGWRY3PoN6UfNd6uwXWTr8PHy4dsm83c2LkX9T47j+xk55GdPNvpWXrV68VrP73GrpRdfHXjVwxtMZR6Fevx/tr38/fffXQ38cfjeeSqR1gyZAmvdXsNP28/Xuv2GltHbeXqGldzd8TdhAWF0a12NwAeueoR3un1TpEVs4uh05oiIiJyWWw/vJ1Z22dxd8TdXN/gelpVb0XExxF8sekLagTVoGPNjoV6nxV7VwBwTa1rGNx8MIviFhHgE8A1ta4B4L7I+3hkwSNsOLiBuTvnsnzv8tz9a+ZubxvWlsOPHaasf1kAVg1flb+IYFSbUQT4BNCzXs/L+t0vJ53WFBERkYuWlZNFdk42/j7++WPj1o/j7u/uZtuobTQMbgjA3bPvZtwv4wBYMngJXWp3ueB7D/l2CHN3zuW3R3/DGPOn7UfSj1DznZo0rNSQ6MRoAIJLBXPo0UMF7u8udFpTREREXObheQ8T8noI/175bzKzMwFYtX8VwaWCaVCpQf5+z3d+njF9xlC1TFVe+fGVC75vZnYmK/auoGPNjucsWhUDK/JUh6eITowmyD+IT67/hI/6fOTWxexiqJyJiIjIRZu/az7GGJ5e8jTtPm3H7O2zWbl3Je1rtD+rJIUGhXJv5L38o90/WBi3kA0HNwDw4boP+WbLN2e959cxX1PptUrsObqHzjU7n/fzH77qYTqEd+CVrq9wV8RdDGg84LJ/R6fomjMRERG5KMlpyew8spNXur5CvYr1eHj+w/Sd0heAe1rdU+AxIyJG8OTiJ/k65msql67MA3MfwFpLy2ot+SXxFwAslqvCruL/2v4f/Rv1P2+GAJ8AVg5beXm/mJtQORMREZGLsvbAWgDahralS+0u9G3Yl/m75vNL4i8MazmswGMqBlakY82OzNo+i7L+ZcmxObQJbcOR9CM83v5x/Lz9qBhYkXsj7yXAJ6Aov47bUTkTERGRi7LmwBq8jBeR1XOvbff19uW6BtdxXYPrzntcv4b9eGj+Q7y1+i061ezEsiHLSsx1YpeTrjkTERGR87LWsvbAWnJsDjO2zmD8L+NpEtIk/1YVhdX/iv74e/sT4BPAi11eVDE7B5UzERERD5aZncl/fvwPyWnJBW5Py0zjxmk30nZcW15a8RK3fnMrpf1K83bPty/6s8LLhXPgHwfY89AeOtXs9Fejl1g6rSkiIuLB5u+azxOLnyA5PZnXur/2p+0frP2Ab7d9S5XSVXhh+Qvk2BwmD5hMi6otLunzKpWq9Fcjl3iaORMREfEwn0R/wvBZw8nOyeaHnT/kjq3/hGGzhtHxs46MWTcGyJ1Ve3ftu3Sp1YUXOucWs2aVm9G8SnMn45d4KmciIiIe5LVVrzFyzkg+2/AZH6z7gLmxc6ldvjZHTx1l8ubJ7D+2n6eWPEXq6VQeW/hY/jMrb212K1VKV+H+1vfrWjEX02lNERERD/HGT28wetFobml6C0fSj/DogkfJzMlkTJ8xBPoEElk9koMnDtJtUjcixkYQeySWe1vdS+/6vfEyXiQ+kqhiVgRUzkREREqwHJvDC8teoF+jfryw/AWua3AdX/T/gqS0JF5c/iJbD29lwBUDCCkdAsAVIVdQq3wtYo/E8kb3N3jk6kfy30vFrGjoweciIiIlWMyhGJqOaUqFgAqknEph6ZCldK7V+bzHLNi1gJ3JO3UK8zIr7IPPNXMmIiJSgq05sAaAlFMphAWFFeoWFj3q9qBH3R6ujibnoAUBIiIiJczBEwf5YO0HnMg4wZr4NZTzL0ej4Ebc2+pevIz+r9/daeZMRESkhHlh2Qt8FP0Rr/30GgZD27C2zL9jvtOxpJBUn0VERIqRkxknWbF3BdZa4lLiqP9efcb/Mj5/e0Z2BtO2TKNDeAdST6ey99he2oW2czCxXCzNnImIiBQjw2cPZ1rMNNqFtSP1dCqxR2IZ9cMo2oW1o3FIY2ZuncmR9CM82eFJ/Lz96Deln64fK2ZUzkRERIqJiRsnMi1mGv0a9WNH8g52Ju9kTJ8xPLPkGR5d8CiNQxrz5uo3CS8XTvc63fH19uXYE8fw9vJ2OrpcBJeVM2PMeOA64JC1tukZ4w8AfweygO+ttY/njT8JjACygf+z1urkuIiICHDvnHuZs2MOB1IP0Da0LVMHTsXP248cm4OX8eJI+hGeXvI0c2PnMrj5YF7v/jq+3r4AKmbFkCuvOfsc6HXmgDGmC9AXuNJa2wR4I2+8MXAL0CTvmA+NMfpfk4iIeLzsnGwmbpxIcKlg3un5DiuHrcTP2w8gf+Xl/a3vp6xfWcKCwvjgbx9QuXRlJyPLX+SymTNr7QpjTK0/DN8HvGqtPZ23z6G88b7AlLzx3caYWKANsNpV+URERIqD2COxpGel81C7hxjaYmiB+5QPKM+SIUso51+OMn5lijagXHZFvVqzAdDRGLPGGLPcGNM6bzwU2H/GfvF5Y39ijBlpjIkyxkQlJSW5OK6IiIizNv62EYDmVZqfd7/I6pHUr1S/KCKJixX1ggAfoALQDmgNTDPG1AEKejZEgc+VstaOBcZC7uObXJRTRESkyGVmZ3Ig9QC1ytci9XQqfb7qgzEGb+NN45DGTseTIlLU5SwemGFzH+i51hiTAwTnjdc4Y78wIKGIs4mIiDhiWsw0tiZtZcW+FSzdvZRnOz1L3Yp1WblvJQBNQprg7+PvcEopKkVdzr4FrgWWGWMaAH7AYWA28JUx5i2gOlAfWFvE2URERC7IWsvH0R/To24P6lSo85ffLzM7k1E/jOJw2mG8jBdd63TlXyv+RaXASvh6+ZKZk0nzquc/pSkliytvpTEZ6AwEG2PigX8C44HxxphfgQxgSN4sWowxZhqwhdxbbIyy1ma7KpuIiMil+nHfj9z3/X0MbDyQr2/6+i+/38K4hRxOO8wb3d/g6hpX0zasLV0ndmXZnmWMbj+aQycPMajJoMuQXIoLk9uNiqfIyEgbFRXldAwREfEg/ab0Y9b2WXgbb/Y+tJfQoALXrxXa7TNuZ17sPBIfScy/RUZcShz3zLmHsdeNpXaF2pcjtrgBY0y0tTbyQvvp2ZoiIiKFtO/YPmZvn81tzW4jx+Ywbv24v/R+GdkZzN4+mxsb3ZhfzADqVKjDwjsXqph5KJUzERGRQpq7cy4Wy7OdnuWaWtfw9Za/dlpz5d6VnMg4wfUNr79MCaUkUDkTEREppPm75hNeLpyGlRrSr2E/YpJiiD0Se0nvdSLjBN/v/B5/b3+61u56mZNKcaZyJiIiUgiZ2ZksiltEr7q9MMbQt1FfAGZtm3XR77Xh4AaCXwvmnZ/foXOtzpT2K32540oxpnImIiJyASv2rqDpmKakZqTSs15PAGqVr8WVVa7kh9gfLuq9Tmed5q7ZdxHkH0TnWp35e5u/uyKyFGNFfZ8zERGRYuXYqWPcPuN2fL18efnal7muwXX529rXaM+Xm78kx+bkP4S8ICcyTrDh4AZiDsXw+k+vsytlF1/f9DUDGw8siq8gxYzKmYiIyHm8vPJlElIT+HnEz7QObX3WtsjqkYyJGkPskVgaVGpQ4PE7k3fS+8ve7ErZBUDLqi2Zf8d8etTt4fLsUjypnImIiJxDRnYGn234jP6N+v+pmAG0qtYKgOiE6HOWs9tn3M6x08eYMmAKDYMb0rxKc4wp6JHSIrl0zZmIiMg5zNo2i8Nph7kr4q4CtzcOaUyATwBRCQXfEH1H8g7WJazjyQ5PMqjpIFpUbaFiJhekciYiInIOU2OmElo2lO51uhe43dfblxZVW/BT/E9Ya7HWEnsklqycLNYnrufDdR9iMHr8klwUndYUERE5h3UJ6+hYsyPeXt7n3GfAFQN4bOFj3DX7LhJOJDAvdh6BPoGkZ6UD0Klmp7/8iCfxLCpnIiIiBTicdph9x/bx99bnv9XFI1c9QlxKHGOixhDoE8gzHZ8h8UQi7cLakZmdSaeanYoosZQUKmciIiIFWJ+4HoBW1Vuddz9jDB/2+ZDXur+Gr5cv/j7+RRFPSjCVMxERkQJEJ0QDEFEtolD7l/Er48o44kG0IEBERKQA0YnR1K1Ql/IB5Z2OIh5G5UxEROQP0jLTWBS3iA7hHZyOIh5I5UxEROQPpm+ZzrHTxxjWYpjTUcQDqZyJiIj8wSfrP6FexXpaaSmOUDkTERE5Q3RCND/u+5F7Wt2ju/mLI1TOREREzvD6T68T5B/EyFYjnY4iHkrlTEREJM/82Pl8veVr7ml1D0H+QU7HEQ+lciYiIgIcOnmIm6ffTLPKzXjumuecjiMeTOVMREQE+G77dxw/fZzP+n6mG8qKo1TOREREgLmxcwktG0qLqi2cjiIeTuVMREQ8XmZ2JgvjFtKrXi+t0BTHqZyJiIjHm7ltJsdPH6d3vd5ORxHRg89FRMRzbTy4kf+u+S9Tfp1Cq2qt6NOgj9ORRDRzJiIinuu/a/7LF5u+oGPNjnx/2/cE+AQ4HUlEM2ciIuK5th7eSvvw9sy/Y77TUUTyaeZMREQ8krWWrUlbuSL4CqejiJxF5UxERDzSwRMHOXb6mMqZuB2VMxER8UhbD28F4IoQlTNxLypnIiLikbYm5ZUzzZyJm1E5ExERj7T18FbK+pWletnqTkcROYvKmYiIeBxrLQvjFhJRLUJPBBC3o1tpiIiIR/l227fEpcSxI3kHT3d82uk4In+iciYiIh4jMTWRAdMGkGNzqBBQgZsa3+R0JJE/cdlpTWPMeGPMIWPMrwVse9QYY40xwXk/G2PMu8aYWGPMJmNMhKtyiYiI55r862RybA5DWwzljR5vEOgb6HQkkT9x5czZ58D7wMQzB40xNYDuwL4zhnsD9fN+tQXG5P0uIiJyWVhrmbRpEpHVI/ms72dOxxE5J5fNnFlrVwBHCtj0NvA4YM8Y6wtMtLl+BsobY6q5KpuIiHiWzOxMBn49kA0HN3B3xN1OxxE5ryJdrWmMuQE4YK3d+IdNocD+M36OzxsTERE5p5hDMXT6rBNJJ5POu983W79hxtYZvNL1FZUzcXtFVs6MMaWAp4HnCtpcwJgtYAxjzEhjTJQxJiop6fx/GUVEpOR586c3uXv23aRlpjFj6wxW7lvJlF+nnPeYCRsnEF4unMfbP65bZ4jbK8rVmnWB2sDGvL8YYcB6Y0wbcmfKapyxbxiQUNCbWGvHAmMBIiMjCyxwIiJSMmVmZ/LyypdJOZXCrpRd+Rf0T42Zys1NbqZiYEV8vX3POiYhNYEFuxbwZIcn8TK6vae4vyL7X6m1drO1trK1tpa1tha5hSzCWnsQmA0Mzlu12Q44Zq1NLKpsIiJSPCzfu5yUUyl0rd2VpXuWsihuEX7efqzav4qqb1alyhtVeHfNu1j7v3+7z4udR47N4damtzqYXKTwXHkrjcnAaqChMSbeGDPiPLv/AMQBscAnwP2uyiUiIsXXjK0zKOVbivF9x+NtvMnIzuCxqx+jWeVmPHLVI7QJbcOD8x7k+WXP5x+z9sBaygeU1wPOpdhw2WlNa+15/4mSN3v2+2sLjHJVFhERKf4yszP5Zus39K7Xm/By4fSo24O5sXO5qfFNvHTtSwDk2ByGzRrGSytfonf93rQLa8faA2tpXb21TmlKsaH/pYqIiFuz1jJz60wmbZrEoZOHGNx8MACPt3+cG6+4kSaVm+Tv62W8eK/3e4SWDeWBuQ+QnpnOpt820Sa0jVPxRS6aHt8kIiJuKT0znfm75mOt5cZpNwJQuXRletfrDUDnWp3pXKvzn44L8g/i8faP88DcBxj/y3iybbbKmRQrmjkTERG3kp6ZTkp6Cv9c9k/6T+3PoOmDCAsKo4xfGe5pdc+fVmMW5LZmt+Hv7c8Ti5/AYFTOpFjRzJmIiLiV0YtG897a9wj0CaROhTrEpcTxTs936FWvV6GfhVkxsCIDGg9gyq9TeP9v71O1TFUXpxa5fMyZy42Lm8jISBsVFeV0DBERuYzC3w5n//H9eBtvYu6PIaR0CBUDK170+6SeTiX+eLxWaYrbMMZEW2sjL7SfTmuKiIjbyMzO5LeTvzGq9Si2jNpCw+CGl1TMAMr6l1Uxk2JJ5UxERNzG9uTtZGRncFXYVTSo1MDpOCKOUDkTERG3YK1l48GNADSv2tzhNCLO0YIAERFxnLWWDp914Kf9PwHQsFJDhxOJOEczZyIi4pjfF6Vt+m3TWcWsMLfLECmpNHMmIiKOiDkUQ5txbVg+dDkzts7A23izesRqQkqHOB1NxFEqZyIiUiSycrKw1uLj5UOOzWFu7FzSMtP4JPoTFsYtpFudbrQObe10TBHHqZyJiEiRGDhtIMdOH6NHnR68u/ZdmlVuBsAn6z/BYnm9++sOJxRxDypnIiLictk52SyKW8TJzJOs2LuCHJvDwRMHqV62OgmpCXSq2Ykbr7jR6ZgibkHlTEREXC4mKYaTmSfxNt54GS9Cy4ay//h+nuzwJDuSdzCq9SiMMU7HFHELKmciIuJyP8f/DMAPt/+Av7c/6xPX848F/6BbnW78vc3fHU4n4l5UzkRExOXWxK+hUmAlutfpjjGGDuEd6FSzE42CGzkdTcTt6D5nIiLiUjk2h+V7l9M2rG3+qUtvL29aVW/lcDIR96RyJiIiLjX116nsStnFHc3ucDqKSLGgciYi4qEmbZzEvXPuZV7sPJd9hrWW55Y9R/MqzRnUdJDLPkekJNE1ZyIiHuj46eMMmzWMbJtNTFIMver1csnnxKXEEXsklo/6fISX0XyASGHob4qIiAdavmc52TabOhXqsP3wdpd9TlRCFABtw9q67DNEShqVMxERD7R492ICfAIY1mIYSWlJpKSnuORzohKi8Pf2p0lIE5e8v0hJpHImIuKBFsUtokN4B5pXaQ7A9uSLmz1LSE3AWnvB/aISo2hRtQW+3r6XlFPEE6mciYh4mL1H9xKTFEO32t1oGNwQ4KJObS6OW0zoW6HcOO1GTmacPOd+OTaH6IRoIqtH/uXMIp5E5UxExMN8tfkrAG5ucjO1y9fGx8snf+Ysx+Zc8PjvdnyHwfDttm+ZGjP1nPst27OM1IxU2obqejORi6FyJiLiQay1TNo0iQ7hHahdoTa+3r7UrVCXmKQYHl3wKGFvhZGYmsjzy54n6WRSge+xKG4RXWp3wdfLlx3JO875OU8seoIaQTW4qclNrvxKIiWObqUhIuJB3lr9FlsPb+Xj6z7OH2tWpRnTt0zP/3nYrGHM3zWf9Mx0/tP9P0Bu2Vq6Zykxh2KISYphSPMhHDh+gNgjsQV+zqK4RaxLWMenN3xKgE+Aa7+USAmjciYi4iHm7JjDowsfZWDjgQxtMTR/fEyfMfSs2xM/bz+eXvI083fNB+CzDZ/x4rUv4uvly+0zbmfyr5Pzj+lapysr9q1g55GdBX7WjK0zKO1bmlub3urS7yRSEqmciYh4iI+jP6ZamWpMHjAZH6///ec/uFQwd0XcBcC6A+t4f937tA1ty5oDa/hmyzekZaYx+dfJPNH+Ca5rcB3bDm+jZdWW1KtQjyW7l2CtxRjD3qN7OZV1ivqV6jN7x2x61etFoG+gU19XpNhSORMR8QAHTxxk7s65PHr1o2cVsz8a3Hww07dOZ0K/CfSd0peXVr7EgeMH6FyrMy93fRkv40X78PYA1KtYj7TMNBJPJDJ351xG/TCKcgHlmDloJgmpCfRr1K+ovp5IiaIFASIiHuCbLd+QbbMZ0nzIefdrHdqaxEcSaRjckNHtR7MlaQsnMk7wwd8++NPjl+pXqg/AjuQdPLXkKcoFlOPQyUPcO+de/L396VO/j8u+j0hJpnImIuIBNh/aTIWACjQKblToY26/8nYaBTfi/9r+H41DGv9pe72K9QD4OuZrDp08xItdXqS0b2k2H9rM7c1up0JghcuWX8ST6LSmiIgH2J68nUbBjTDGFPoYP28/tty/5Zzba5arSdUyVfk4OnflZ9+GfVmyewlTY6byYLsH/3JmEU+lmTMREQ+w7fC2/KcBXAxjzDkLnbeXN692fZVsm01EtQiqlKnCS9e+xOd9P+fKKlf+1cgiHkszZyIiJdyxU8c4eOIgjSoV/pRmYd3Z/E6W7FnCtbWuBXJPdf5+ulNELo3KmYhICff7o5kuZebsQryMFxP6Tbjs7yviyXRaU0SkhPv9oeYXsxhARJzjsnJmjBlvjDlkjPn1jLHXjTHbjDGbjDEzjTHlz9j2pDEm1hiz3RjT01W5REQ8zdbDW/Hx8qFuhbpORxGRQnDlzNnnQK8/jC0EmlprrwR2AE8CGGMaA7cATfKO+dAY4+3CbCIiHmPBrgVEVo/E19vX6SgiUgguK2fW2hXAkT+MLbDWZuX9+DMQlve6LzDFWnvaWrsbiAXauCqbiEhJk5Gdwezts7HW5o8dP32cHck7iE6Mpn+j/g6mE5GL4eQ1Z8OBuXmvQ4H9Z2yLzxv7E2PMSGNMlDEmKikpycURRUSKh2kx0+g7pS8/7vsxf6z7pO40/bApgMqZSDHiSDkzxjwNZAFf/j5UwG62gDGstWOttZHW2siQkBBXRRQRKVa2JOXeLHZe7DwAktOSWXtgLdk2mzahbfIftSQi7q/Iy5kxZghwHXC7/d/8ezxQ44zdwoCEos4mIlJc7UjeAcCCuAUA/LT/JwDm3T6PRXcuciyXiFy8Ir3PmTGmFzAauMZam3bGptnAV8aYt4DqQH1gbVFmExEpTvYc3cPO5J2sjl+Nj5dP/r3MohOimfrrVFbsXYGftx8da3YkwCfA4bQicjFcVs6MMZOBzkCwMSYe+Ce5qzP9gYV5jwP52Vp7r7U2xhgzDdhC7unOUdbabFdlExEp7npM6sHOIzvzf/b18qVbnW4silvELd/cAsDVNa5WMRMphlxWzqy1txYw/Ol59n8ZeNlVeURESoqDJw6y88hOHmz7IF1rd+WGKTeQmZPJzY1vZtz141ifuJ6Rc0bSr2E/p6OKyCXQ45tERIqZ1ftXAzCoySDahbUjvFw4+47to2FwQ2qWr0nN8jXp26gvpsC1ViLi7vT4JhGRYmZ1/Gr8vP2IqBaBMYYbGtwAQMNK/3t2ppfxIu/yEREpZjRzJiJSzKyOX03Lqi3x9/EH4OlOT3NVjauoUqaKw8lE5HLQzJmISDHyxaYvWL1/NR3DO+aPVS1Tldua3eZgKhG5nFTORESKic83fM6dM++kU81OPN3paafjiIiL6LSmiEgxEHMohhGzR9C9Tne+u/W7/FOaIlLyaOZMRKQYmBYzDYAvbvxCxUykhFM5ExEpBr7d/i0dwjtQuXRlp6OIiIupnImIuLndKbvZ9Nsm+jbs63QUESkCKmciIm5uwa7ch5lf1+A6h5OISFFQORMRcXNrD6ylUmAl6les73QUESkCKmciIm5uXcI62oS20R3/RTyEypmIiBs7mXGSmKQYWldv7XQUESkiKmciIm5sfeJ6cmwOrUNVzkQ8hcqZiIgbW3tgLYBmzkQ8iMqZiIgbW7V/FXUq1NFDzUU8iMqZiIibstby0/6faF+jvdNRRKTiHxtCAAAgAElEQVQIqZyJiLipuJQ4fjv5m8qZiIfRg89FRNzQV5u/4otNXwDQPlzlTMSTqJyJiLiZ01mn+fsPfyflVAqVAivROKSx05FEpAipnImIuJnZ22eTciqFj/p8RPvw9ngZXYEi4klUzkRE3MznGz8nLCiMuyLuwtvL2+k4IlLE9M8xERE3kp2TzbI9y+jfqL+KmYiHUjkTEXEjO5J3kJaZRmT1SKejiIhDVM5ERNxIdGI0ABHVIhxOIiJOUTkTEXEj6xPXE+ATQKPgRk5HERGHFGpBgDEmBLgbqHXmMdba4a6JJSLimdYnrqd5leb4eGm9loinKuzf/lnASmARkO26OCIinisrJ4tfDv7CbU1vczqKiDiosOWslLV2tEuTiIh4IGstK/auYMGuBbSs1pLjp4/TvW53p2OJiIMKW87mGGP+Zq39waVpREQ8yLFTx7jru7uYvmU6AKV8S1HGrwy96/V2OJmIOKmwCwIeJLegnTLGpOb9Ou7KYCIiJVVcShzzY+fTdWJXZm6dyb+v/TfDWwwnLTONGxreQKBvoNMRRcRBhZo5s9aWdXUQERFPMP6X8dz//f2czj6Nn7cfs26ZRZ8GfTh08hAxSTHcH3m/0xFFxGHGWlu4HY25AeiU9+Mya+0cl6UqpMjISBsVFeV0DBGRQrHWUvG1ilwRfAUvdnmRGuVq0KBSA6djiUgRMcZEW2sveIfpwt5K41WgNfBl3tCDxpgO1ton/kJGERGPkngikaOnjnJbs9voWqer03FExE0VdkHA34AW1tocAGPMBOAXQOVMRKSQYg7FANAkpInDSUTEnV3MEwLKn/G63OUOIiJS0m1J2gJAk8oqZyJyboWdOXsF+MUYsxQw5F579qTLUomIlEAxSTFUCqxESKkQp6OIiBsr1MyZtXYy0A6YkffrKmvtlPMdY4wZb4w5ZIz59YyxisaYhcaYnXm/V8gbN8aYd40xscaYTcYYPfFXREqcLUlbaFK5CcYYp6OIiBs7bzkzxjTK+z0CqAbEA/uB6oUoUJ8Dvf4w9gSw2FpbH1jM/65Z6w3Uz/s1EhhT+K8gIuL+rLXEJMXQOLix01FExM1d6LTmP8gtS28WsM0C157rQGvtCmNMrT8M9wU6572eACwDRueNT7S59/X42RhT3hhTzVqbeIF8IiLFwtbDWzl66iiR1S+4il5EPNx5y5m1dmTey97W2lNnbjPGBFzC51X5vXBZaxONMZXzxkPJnZH7XXze2J/KmTFmJLmFkfDw8EuIICJS9BbuWghAtzrdHE4iIu6usKs1fyrk2KUq6AKMAu+Oa60da62NtNZGhoTooloRKR4W7V5EvYr1qFm+ptNRRMTNnXfmzBhTldwZrEBjTEv+V6KCgFKX8Hm//X660hhTDTiUNx4P1DhjvzAg4RLeX0TE7WRmZ7JszzLuaHaH01FEpBi40DVnPYGh5Jalt84YTwWeuoTPmw0MAV7N+33WGeN/N8ZMAdoCx3S9mYiUFK+teo0TGSfo26iv01FEpBi40DVnE4AJxpgB1tpvLuaNjTGTyb34P9gYEw/8k9xSNs0YMwLYB9yUt/sP5D6FIBZIA4ZdzGeJiLirrUlbeX758wxqMohe9f64gF1E5M8KdRNaa+03xpg+QBMg4Izxf53nmFvPselPD5TLW6U5qjBZRESKk9d/eh0/bz/e/9v7TkcRkWKiUAsCjDEfAYOAB8i97uwmQFe1ioicx8ETB/ly85cMazGM4FLBTscRkWKisKs1r7bWDgZSrLUvAFdx9gX8IiLyBxM3TiQjO4MH2z7odBQRKUYKW85+v8dZmjGmOpAJ1HZNJBGR4uuheQ/x2ILHAJi1fRYR1SKoX6m+w6lEpDgpbDn7zhhTHngdWA/sASa7KpSISHGUnZPN+F/G8/6699l1ZBer96/mhgY3OB1LRIqZCy4IMMZ4kfs8zKPAN8aYOUCAtfaYy9OJiLiphNQE3vzpTSyWf3X5F2X8yrDxt42kZqQCMGL2CCxWt88QkYt2wXJmrc0xxrxJ7nVmWGtPA6ddHUxExJ1N2jiJt37Ovf1jWFAY/7jqH6zYuwKAioEVWb53OR3DO9K8SnMnY4pIMVTY05oLjDEDjDEFPWZJRMTjbEveRrUy1egQ3oH31r5Hdk42K/etpHb52rzV4y1GtBzB3Nvnov9sisjFKtR9zoB/AKWBLGPMKXJvp2GttUEuSyYi4sa2Hd5Go+BG3N/6fm76+iZajW3F5kObGdFyBENaDGFIiyFORxSRYqpQM2fW2rLWWi9rrZ+1NijvZxUzEfFI1tr8cta/UX9e7foqAT4BPNj2QV7r/prT8USkmCvUzJkxplNB49baFZc3joiI+zt08hBHTx2lUXAjvL28Gd1hNKM7jHY6loiUEIU9rfnYGa8DgDZANHDtZU8kIuLmth3eBkCj4EYOJxGRkqiwz9a8/syfjTE1AM3di4hHUjkTEVcq7GrNP4oHml7OICIixUFKegrvrX2P4FLBhAWFOR1HREqgwl5z9h5g8370AloAG10VSkTEXY1eNJodyTuYe/tcvMyl/vtWROTcCnvNWdQZr7OAydbaVS7IIyLi1lbtX0Xv+r3pWqer01FEpIQq7DVnE4wxIXmvk1wbSUTEPaVlprHt8DYGXjHQ6SgiUoKdd07e5HreGHMY2AbsMMYkGWOeK5p4IiLuITktmfWJ68mxObSo2sLpOCJSgl1o5uwhoD3Q2lq7G8AYUwcYY4x52Fr7tqsDiog4zVpLy49bkpaZBqByJiIudaGrWQcDt/5ezACstXHAHXnbRERKvB3JO9h/fD/J6ckE+QdRq3wtpyOJSAl2oXLma609/MfBvOvOfF0TSUTEvfy0/ycAAn0CiagWoYeZi4hLXei0ZsYlbhMRKTFW7V9FxcCKLBuyjEDfQKfjiEgJd6Fy1twYc7yAcUPuY5xEREq8VftXcVXYVTSr0szpKCLiAc5bzqy13kUVRETEHaWkp7Dt8DbuvPJOp6OIiIfQ7a1FRM4jKiH3HtxtQts4nEREPIXKmYjIefxezlpVa+VwEhHxFCpnIiLnsS5hHfUq1qNCYAWno4iIh1A5ExE5j3UJ62hdvbXTMUTEg6iciYicw8ETB4k/Hk9k9Uino4iIB1E5ExE5h2V7lgHQvkZ7Z4OIiEdRORMROYeFuxZSPqC8Zs5EpEipnImInCHmUAxzd87FWsuCuAV0rd0Vby/d8lFEis6FnhAgIlLiffbLZyzevZhjp4+xYNcCsnKymHPrHOKPx/Nsp2edjiciHkYzZyLi0b7a/BXDZw9n2Z5l7D+2n36N+gFw8/SbCfAJoE/9Pg4nFBFPo5kzEfFYaw+sZeR3I+kQ3oElg5fg6+0LQP+p/fl227f8q/O/CA0KdTiliHgalTMR8UjL9yznxmk3UqVMFaYNnJZfzABe6PwClUtV5rH2jzmYUEQ8lU5riojHiU6IpuvErgSXCmbhnQupVrbaWduvrHIlH1//MQE+AQ4lFBFPppkzEfE4c2Pnkm2z+XHYj4SUDnE6jojIWTRzJiIeZ+2BtTQKbqRiJiJuyZFyZox52BgTY4z51Rgz2RgTYIypbYxZY4zZaYyZaozxcyKbiJRs1lrWHlhLm9A2TkcRESlQkZczY0wo8H9ApLW2KeAN3AL8B3jbWlsfSAFGFHU2ESn59h/fz28nf6NNdZUzEXFPTp3W9AECjTE+QCkgEbgWmJ63fQLQz6FsIlKCrT2wFkAzZyLitoq8nFlrDwBvAPvILWXHgGjgqLU2K2+3eKDAmwsZY0YaY6KMMVFJSUlFEVlESpDZ22dTzr8cV1a50ukoIiIFcuK0ZgWgL1AbqA6UBnoXsKst6Hhr7VhrbaS1NjIkRBfzikjhpZ5O5Zut3zCoySD8ffydjiMiUiAnTmt2A3Zba5OstZnADOBqoHzeaU6AMCDBgWwiUoJN3zKdtMw0hrQY4nQUEZFzcqKc7QPaGWNKGWMM0BXYAiwFBubtMwSY5UA2ESnBlu9dTtUyVbkq7Cqno4iInJMT15ytIffC//XA5rwMY4HRwD+MMbFAJeDTos4mIiXbrpRdNKjUgNx/F4qIuCdHnhBgrf0n8M8/DMcBWj4lIi6z68guetbr6XQMEZHz0hMCRMQjpGWmkXgikboV6jodRUTkvFTORMQj7E7ZDaByJiJuT+VMRDxCXEocAHUq1HE4iYjI+amciUiJ93P8z8zcNhOAuhU1cyYi7s2RBQEiIkXFWku/Kf347eRvAFQKrORwIhGR89PMmYiUaDFJMfnFrGXVlrqNhoi4Pc2ciUiJtjhuMQCb79tMtTLVHE4jInJhKmciUqIt2bOEuhXq0rRyU6ejiIgUik5rikiJlZaZxtLdS+lau6vTUURECk3lTESKpdX7V9N1Ylc+XPfhOff5Zss3pGakcmuzW4swmYjIX6PTmiJSrKw7sI4P1n3AhI0T8PP2Y8nuJZT1K8udze/8077jfhlHvYr1uKbmNQ4kFRG5NJo5E5Fi45st39BmXBsm/zqZ0e1Hk/hIIp1rdebe7+9lZ/LOs/bdkbyDFXtXcFfLu7RCU0SKFZUzESk2Plj3AXUq1OHgIwd5tdurVAysyBf9v8Df25+Rc0YCudeZAYxbPw5v482QFkOcjCwictFUzkSkWNidspule5YytPlQKgRWyB8PDQrl+c7Ps2zPMnp/2ZvKr1dm5d6VTNg4gesbXk/VMlUdTC0icvFUzkTE7WVkZ/DgvAfxMl4FzoTdFXEXwaWCmRc7j9PZp+n0eScOnTzEqNajHEgrIvLXaEGAiLitEbNG0KRyExJSE/hux3e83/t9wsuF/2m/Ur6leKfnO8zbNY8H2z7If1b9h1GtR9G5VueiDy0i8hcZa63TGS5ZZGSkjYqKcjqGiLhAWmYaQa8EUT6gPABd63Rl6sCpDqcSEbl0xphoa23khfbTzJmIOGJL0hbeWv0WaZlp3B1xN11qdzlr+/rE9WTbbJLTkwEYfOVgJ2KKiBQ5lTMRKXI5Nofuk7pz/PRxAn0CmbV9Fgf+cSB/lgxgTfwaACoGVsTbeNOjbg+n4oqIFCktCBCRIrf5t80kpCbwfu/3mXv7XNIy05i0cdJZ+6w5sIZa5WsxdeBUJvafiK+3r0NpRUSKlsqZiBS5xbsXA7nXkbWq3oo2oW34YN0HnM46jbWWGVtnsGzPMtqGtqVbnW70qtfL4cQiIkVH5UxEityiuEU0Cm5EWFAYAE93fJrtydsZMXsEn/7yKQOmDSArJ4vbmt3mcFIRkaKnciYiLncq6xQLdi0gOyeb1NOprNi7gq61u+Zvv6HhDbzU5SW+3Pwld393Nx3CO5D0WBI3NLzBwdQiIs7QggARcblnljzDm6vfJLJ6JFeHXc3JzJMMbn726sunOz1Ny2otmbBxAv/p9h+8vbwdSisi4izd50xEXCohNYG679alVbVWbEnaQsqpFLrV6cbCOxc6HU1EpEjpPmci4hbeXv02mdmZTOg3gVNZp3h4/sO80vUVp2OJiLgtlTMRcZmM7Aw+3/g5fRv1pW7FugAsuHOBw6lERNybFgSIiMvM3j6bw2mHuavlXU5HEREpNlTORMQltiZt5aF5D1GzXE3d3V9E5CKonInIZffi8hdp+XFLsnKy+O7W77TyUkTkIqicichlNX3LdJ5b9hw3NLyBqJFRNKvSzOlIIiLFihYEiMhlc/z0ce6dcy+R1SP58sYv9TxMEZFLoHImIpfNmHVjSE5PZt4d81TMREQukU5rishlkZ6Zzls/v0WPuj2IrH7BeyyKiMg5aOZMRP6SY6eO8eO+Hzl08hCHTh5idPvRTkcSESnWVM5E5JKkZ6YTfzyeEbNHsHLfSoL8g2gc0pgutbo4HU1EpFhTORORSzLw64H8sPMHAJqENCEmKYZRrUdhjHE4mYhI8eZIOTPGlAfGAU0BCwwHtgNTgVrAHuBma22KE/lE5PyW71nODzt/4O6IuxnUZBAR1SL4cvOXDG853OloIiLFnrHWFv2HGjMBWGmtHWeM8QNKAU8BR6y1rxpjngAqWGvPe/FKZGSkjYqKKoLEInKmnl/05NdDvxL7QCyBvoFOxxERKRaMMdHW2guumCry1ZrGmCCgE/ApgLU2w1p7FOgLTMjbbQLQr6iziciFZeVksXLvSm5qfJOKmYiICzhxK406QBLwmTHmF2PMOGNMaaCKtTYRIO/3ygUdbIwZaYyJMsZEJSUlFV1qEQHg10O/kp6VTtvQtk5HEREpkZwoZz5ABDDGWtsSOAk8UdiDrbVjrbWR1trIkJAQV2UUkXNYE78GgLZhKmciIq7gRDmLB+KttWvyfp5Obln7zRhTDSDv90MOZBORC1hzYA3BpYKpXb6201FEREqkIi9n1tqDwH5jTMO8oa7AFmA2MCRvbAgwq6izicj55dgcVu1fRZvQNrplhoiIizh1n7MHgC/zVmrGAcPILYrTjDEjgH3ATQ5lE5FzGBs9lh3JO3iqw1NORxERKbEcKWfW2g1AQUtJuxZ1FhEpnKOnjvL4wsfpVqcbg5sPdjqOiEiJpQefi0ihfLHpC1IzUnm166s6pSki4kJ6fJOInFdiaiKLdy9mTNQYWlVrRavqrZyOJCJSoqmcicg5Tdo4ieGzh5OVkwXA2OvGOpxIRKTkUzkTkQJt/m0zI+eM5OoaV/N2z7c5lXWKdmHtnI4lIlLiqZyJeIhJGyfROrQ1jYIbFWr/p5c8TRm/MkwbOI0qZaq4OJ2IiPxO5UzEAyyOW8zgbwfTtHJTBl85mNSMVF7o/MI5L+zfe3Qvc3bM4amOT6mYiYgUMZUzkRIuKyeLB+c9SBm/Mvx66FceX/Q4AKFlQxnZamSBBe2jqI8wxjCy1ciijisi4vF0Kw2REm7MujHEJMUwsd9E+jbsy5DmQ+hSqwv3fn8v9d+rz4aDG8jKyeKt1W+x5+gedqfs5p017zCw8UDCy4U7HV9ExOMYa63TGS5ZZGSkjYqKcjqGiFv4KOojagTVoE+DPvljKekp1Hm3Dq2rt2b+HfPzZ8lOZJzg65iveW7Zcxw9dZSO4R2ZGzuXLrW64Oftx4/7fmTrqK3UKFfDqa8jIlLiGGOirbUF3YT/LDqtKVICJKYm8sDcB6hetjpx9eLw9vIGYF7sPI6eOsq/uvzrrNOXZfzKMKzlMHrU7cEdM+9gbuxcmoQ0YemepQB81OcjFTMREYeonImUAOPWjyMrJ4t9x/Yxf9d8/lb/bwAs27OMIP8gWldvXeBxoUGhLB68mOiEaJpVacbfvvwbHcM7ck/kPUUZX0REzqByJlLMTfl1Cv9d81861+rMlqQtjI0e+79ytncZnWp2yp9JK4iX8aJ1aG55WzJkSZFkFhGRc9OCAJFibGfyTm795lbCgsJ4t9e7DGoyiPm75pOemU5iaiI7knfQuWZnp2OKiMhFUDkTKcZ+v0Zs6sCpNKvSjN71enMq6xT/Xvlvrvr0KgC61enmZEQREblIKmcixZC1lvjj8Szfu5yqZarSoFIDADrX6kyATwAvrXwJYwyzb5lN86rNHU4rIiIXQ+VMpBj6fMPn1Hi7BjO2zuCamtfkr8QM9A2kc63OAIy/YTzXN7zewZQiInIptCBApBj6OPpjAE5lncovY797+dqXubnxzXSp3cWBZCIi8lepnIkUM5t/28yaA2t4/OrHOXrqKAOuGHDW9ohqEURUi3AonYiI/FUqZyLFhLWWZ5c+y+s/vU6gTyCPXP0IlUtXdjqWiIhcZrrmTKSYeGLRE7y88mUGNh7IurvXqZiJiJRQmjkTKQZOZpzkg3UfcEvTW/ii/xdnPYpJRERKFs2ciRQDM7fN5GTmSe6LvE/FTESkhFM5EykGJm2aRK3ytegQ3sHpKCIi4mIqZyJuLiE1gUVxi7ij2R14Gf2VFREp6fRfehE399Xmr8ixOdzZ/E6no4iISBHQggCRc1i6eylB/kG0rNaSOTvmsGrfKtqFtaNL7S6UDyhfJBlOZpxk/C/jaRvaNv8RTSIiUrKpnIkUYNNvm+j1ZS/K+pWlS+0uTN8yHYPBYgEo61eWd3q9w/CWw12WIT0znfbj27Pt8Dam3TTNZZ8jIiLuRac1pdj5af9PrNq36pKOtdYy5NshPLPkGdIz08+5z7BZwwjyD+L46eNM3zKdZzo+w4mnTrBi6ApevvZlmlZuyn3f38fq/avPOjYqIYrmHzVn8MzBHEk/ckkZf7d492I2/raRif0nMrDxwL/0XiIiUnwYa63TGS5ZZGSkjYqKcjqGXAYbD27k/h/uZ+AVA3n4qofPud/S3Uvp9WUvyviVYc+DeyjrX/aiPmf5nuV0ntAZAF8vX64IuYIbG93Ic9c8l3+Lip/jf+aqT6/ik+s/wdt4s/vobl7o/MJZt7BIOplExNgIDhw/wNMdn+bFa1/k+OnjtBrbipT0FJLTk3n52pd5quNTF/+Hkef+7+9n4saJJD+ejL+P/yW/j4iIuAdjTLS1NvJC++m0pjjKWsubq9/kqcVPkZmTydFTR89ZzhJSExj49UCqlqnKvmP7eHfNuzzV8amLuu/XR9EfUT6gPFMGTGHZnmX8uP9Hnl/+PGX8yhCXEseprFMcPHmQUr6lGNRk0DnLX0jpEDbcs4GH5z/MSytf4nDaYebtmse+Y/tYOmQpD817iPm75l9yObPWMjd2Ll3rdFUxExHxMCpn4hhrLQ/Ne4h3175L/0b9aVCpAf9Z9R8SUxOpVrbaWftmZGcw9NuhpGems3rEav5v7v/xzNJn+Dj6Y4a2GMq2w9tIPJHIA20eYMAVA5i9fTaTNk3i4ImD9KrXi6trXM249eOYvmU6o1qPome9nvSs15OsnCwiPo7g0YWPUsq3FN7Gm9SMVO648o4LzspVKlWJT2/4lH3H9vFR9EdEVItgYr+JdKzZkZ51e/LG6jc4fvo4Qf5BF/yz2HZ4GxM3TiQlPYXOtTpjjGHP0T080f6Jv/RnLCIixY9Oa4pjFu5aSI8vevBwu4d5s8eb/HLwF1qNbcWk/pO448o7ADh26hjvr32fubFzWbV/FZ9c/wl3RdzF0VNHmfLrFCZunMjq+NWElwsnwCeAHck7KONXhhMZJ6hWpho1y9dkTfwaLJbyAeW588o7+VeXf5212nLjwY2MWz+Opzo+ha+3L2+tfovhLYdTr2K9Qn2PjOwM0jPTKRdQLn9s2Z5ldJnQhZmDZtKvUT8A9h7dy3c7vsPXy5fOtTozL3YedSvW5Uj6Ee7//n5OZZ2itF9pjp8+DsCVVa5k2ZBlVAiscLn+yEVExEGFPa2pciaOeXzh47zz8zscfeIopXxLkWNzCHk9hOsbXM/n/T7ncNphuk/qzoaDG6hWphr/7vpvhrYYetZ7WGtJSksipFQImTmZvL7qdZLSkugQ3oH+jfrj7eXNnqN7WL1/NX0a9CnULNblkJGdQbU3q3FNzWuYMWgG323/jttn3E5qRmqB+zev0pwfbv+BKqWrsHLfSrYmbWVIiyGU8i1VJHlFRMT1VM7E7UWOjaS0X2mWD12ePzZ81nCmxkxl30P7eG7pc4xdP5bZt8ymd/3eDia9NE8tfopXf3yV5UOXc/3k66lToQ6TB0zmVNYp5u+aT+96vUlOT8bP249W1Vrp2jIRkRJO5UzcWkp6CsGvB/Nsp2d5vvPz+eNbkrbQ5MMm3HnlnUyNmcrQ5kP5+PqPnQv6Fxw4foBa/61Fjs3B39ufTfdtKvSpUhERKXm0WlPc2oq9K8ixOXSp1eWs8cYhjRlwxQAmbZqEv7f/X7oVhdNCg0L57tbvWLBrAb3r9VYxExGRQlE5k/9v795jqyrTPY5/n5a2QLnUAkWgBUR6BEpQzhCGCBIDzgSBAVRAlBEiRENCRjxHB1FxRk/OOJqZDOpIFCMeuajIwRERjeFSUHpEpNyFqghSQAiUOxaEXp7zx15gxRYQ6F57t79P0uy13vXuvZ7tg+nT913rXaFYtmMZdevUpUdmj58dm3nbTP544x9Jr5dOm7Q2IUR35fRr349+7fuFHYaIiMQRFWcSimU7ltEzq2el11nVT6rPrzN/HUJUIiIi4Qvt8U1mlmhm68xsYbB/jZmtMrOtZva2mSWHFZtUr6LiIjbu2/izKU0REREJ99maE4CCCvvPAlPcPRs4DIwNJSqpNmduPlm+YzkAfa7pE2I0IiIisSmUaU0zywQGAH8B/tMiz9/pA9wddJkBPAm8FEZ8cuW4O9PXTefdL98lb2ceLRu25NipY2SkZtCt5QVvWBEREal1who5ew6YCJQH+02AI+5eGuzvBlpV9kYzu9/M8s0sv6ioqPojlcvyxqY3uO/9+9h6cCt35txJuZdjGEvuWUJSYlLY4YmIiMScqI+cmdlAYL+7rzGzm880V9K10gXY3P0V4BWIrHNWLUHKFbHjyA7GfzieXq17sXz0chITEikrL6O0vFQLroqIiFQhjGnNnsAgM+sP1AUaERlJSzOzOsHoWSawJ4TY5AopKSth1LujcHdm3TaLxIREABITEs9ui4iIyM9FfVrT3R9190x3bwuMAHLdfSSwDBgadBsNvBft2OTyuDuHTh7inS3v0PmlzqzYuYKp/afSNq1t2KGJiIjEjVha5+wRYI6Z/TewDpgecjzyC42aP4rZG2cDkZX+F4xYwO+u+13IUYmIiMSXUIszd18OLA+2twPdw4xHLt2mfZuYvXE2wzoNY3jOcG7rcJumL0VERC5BLI2cSRzadmgbk5ZOIm9nHo1SGvHywJdJr5cedlgiIiJxS8WZXLLi08V0f7U7JWUl9G3XlzE3jFFhJiIicplUnMklW/XdKg6dPMTCuxYy4N8GhB2OiIhIjRDm45skzuXtzMMwerXuFXYoIiIiNYaKM7lkeTvz6NK8C43rNg47FBERkRpD05ryi33+3ecs2raIlbtXMqrLqLDDERERqVFUnMkv8vSKp3k89zZogQcAAAyhSURBVPGz+33b9Q0xGhERkZpHxZlctCXblzA5dzIjOo/ghX4vsK94HznNcsIOS0REpEZRcSYXbcpnU8hslMn0QdOpn1SfZqnNwg5JRESkxtENAXJRjp86zpLtSxjaaSj1k+qHHY6IiEiNpZEzOa9yL2fsgrF8eeBLTpedZkiHIWGHJCIiUqOpOJMqlZSV8PdP/87r618HoEm9JtyYdWO4QYmIiNRwKs7kJ34o/YHZG2dTUFTA6xte59DJQwy6bhCP9nqU0vJS6iTon4yIiEh10m9a+YmHFz3M1NVTSbAEBl83mFHXj6J/dn+SE5PDDk1ERKRWUHEmZ+XtzGPq6qn8ofsf+Ntv/kZKnZSwQxIREal1dLemAODuTFw8kVYNW/HXvn9VYSYiIhISjZwJAO9//T4rd69k2sBppCanhh2OiIhIraWRM2HBVwu465276Ni0I/fecG/Y4YiIiNRqGjmrZQqPFPL8qucpOFBAu7R2FB4t5IOtH9D16q58OPJDkhKTwg5RRESkVlNxVoss2raIwXMG4+50aNqBTwo/ISM1gz/1/hOP3fSYrjMTERGJASrOarCdR3fyeO7jrN27lqxGWeTvySc7PZsP7v6ArMZZuDtmFnaYIiIiUoGKsxpq8bbFDJ83nFOlp/jttb9lc9FmyryMecPnkdU4C0CFmYiISAxScVYDHTxxkMFzBtM+vT3zR8yn3VXtcHdKyku0mKyIiEiM092acWjelnks3b4Ud6/0+PR10zlZepI373iTdle1AyKjZCrMREREYp9GzuLM9sPbGfa/wwBontqc/tn96dC0AwdPHGRiz4nUS6rHS/kv0btNbzpndA45WhEREfmlVJxdQElZSUwtLzF742wM45+3/pOVu1cyd/NcikuKSbAEXlv/Go1TGlN4pJCp/aeGHaqIiIhcAk1rnsfavWu59oVrWf3d6rBDASKPWJq5YSY3t72Z8d3HM/v22ex9aC+7/mMX+ffl0+eaPjSt35SFdy+kf3b/sMMVERGRS6CRs/Non96ek6UneWTJIywdtfS8dzceP3Wc1ORUEqz66t3Pdn/GtsPbmNx78tm2hikNaZjSkMxGmbw99O1qO7eIiIhEh4qz82iU0ognej/BhI8m8NCih3B3SstL6dqiK12v7kpORg7JicnsOb6HDi92IK1uGvd0uYfx3cfTsmHLC37+0R+O8umuT9m0fxMb9m3g6T5P0yatTZX9Z22cRb069bij4x1X8muKiIhIDFFxdgHjuo3j48KPeX7V8yQnJlMnoQ4vrn4RgGb1m7Hi3hVMWzONEyUn6JHZg2f+7xnmfzWfDeM2kGiJlHs5iQmJQGRackvRFopLipm5YSavrn2VU2WnAEhKSGLt3rVMvmkybdLacOzUMVo3bk1OsxxKykso93LmfDGHIR2G0DClYWj/PURERKR6WVXLMcSDbt26eX5+flTOdfDEQeon1SelTgrfHPqGNXvW8MBHD5CalMq+4n0MzxnOjCEzeLfgXW6feztDOw1l8/7NFB4tZNyvxrG8cDlFxUXsOrYLgARLYMwNYxjZZSQdm3bkq4NfMeDNAXx/+vuz5zSMFg1bcODEAZqnNmfXsV0suWcJfdv1jcp3FhERkSvHzNa4e7cL9lNxdulyv81lzHtj6NSsEy8PfJnWjVvj7twy6xZyv83lptY3kZSYRO63uVzf/HpyMnLomdWTlg1bkp2eTU5Gzk8+72TJSQqPFrLjyA5Sk1L5cOuHfH3oa1o0aMGWoi082ONBBl03KKRvKyIiIpdDxVmIik8Xc/z0ca5ucDXlXs4X+7+gc0bnar1ZQERERGLbxRZnuuasGqQmp5KanApEpi+7NO8SckQiIiISLzSUIyIiIhJDVJyJiIiIxBAVZyIiIiIxJOrFmZllmdkyMysws81mNiFoTzezxWa2NXi9KtqxiYiIiIQtjJGzUuAhd+8I9ADGm1knYBKw1N2zgaXBvoiIiEitEvXizN33uvvaYPs4UAC0AgYDM4JuM4Ah0Y5NREREJGyhXnNmZm2BrsAqoLm774VIAQdkVPGe+80s38zyi4qKohWqiIiISFSEVpyZWQPgHeBBdz92se9z91fcvZu7d2vWrFn1BSgiIiISglCKMzNLIlKYveHu/wqa95lZi+B4C2B/GLGJiIiIhCmMuzUNmA4UuPs/KhxaAIwOtkcD70U7NhEREZGwhfH4pp7APcAmM1sftD0GPAPMNbOxwE5gWAixiYiIiIQq6sWZu+cBVsXhvtGMRURERCTW6AkBIiIiIjFExZmIiIhIDFFxJiIiIhJDzN3DjuGSmVkRUBiFUzUFDkThPFI9lL/4pxzGP+Uwvil/V0Ybd7/gIq1xXZxFi5nlu3u3sOOQS6P8xT/lMP4ph/FN+YsuTWuKiIiIxBAVZyIiIiIxRMXZxXkl7ADksih/8U85jH/KYXxT/qJI15yJiIiIxBCNnImIiIjEEBVn52Fm/czsKzP7xswmhR2PVM7MXjOz/Wb2RYW2dDNbbGZbg9ergnYzsxeCnG40s38PL3IBMLMsM1tmZgVmttnMJgTtymGcMLO6Zva5mW0IcvhU0H6Nma0Kcvi2mSUH7SnB/jfB8bZhxi8/MrNEM1tnZguDfeUwBCrOqmBmicBU4FagE3CXmXUKNyqpwutAv3PaJgFL3T0bWBrsQySf2cHP/cBLUYpRqlYKPOTuHYEewPjg/zXlMH6cAvq4+/XADUA/M+sBPAtMCXJ4GBgb9B8LHHb39sCUoJ/EhglAQYV95TAEKs6q1h34xt23u/tpYA4wOOSYpBLu/glw6JzmwcCMYHsGMKRC+0yP+AxIM7MW0YlUKuPue919bbB9nMgvhlYoh3EjyMX3wW5S8ONAH2Be0H5uDs/kdh7Q18wsSuFKFcwsExgAvBrsG8phKFScVa0VsKvC/u6gTeJDc3ffC5Ff/kBG0K68xrBgaqQrsArlMK4E02Hrgf3AYmAbcMTdS4MuFfN0NofB8aNAk+hGLJV4DpgIlAf7TVAOQ6HirGqV/QWgW1vjn/Iao8ysAfAO8KC7Hztf10ralMOQuXuZu98AZBKZeehYWbfgVTmMMWY2ENjv7msqNlfSVTmMAhVnVdsNZFXYzwT2hBSL/HL7zkx1Ba/7g3blNQaZWRKRwuwNd/9X0KwcxiF3PwIsJ3L9YJqZ1QkOVczT2RwGxxvz80sTJLp6AoPMbAeRy3j6EBlJUw5DoOKsaquB7OBOlWRgBLAg5Jjk4i0ARgfbo4H3KrSPCu746wEcPTN1JuEIrlOZDhS4+z8qHFIO44SZNTOztGC7HnALkWsHlwFDg27n5vBMbocCua5FN0Pl7o+6e6a7tyXy+y7X3UeiHIZCi9Ceh5n1J/KXQyLwmrv/JeSQpBJm9hZwM9AU2Af8GZgPzAVaAzuBYe5+KCgEXiRyd+cJ4F53zw8jbokws17ACmATP17r8hiR686UwzhgZl2IXByeSOSP/rnu/l9m1o7IKEw6sA74vbufMrO6wCwi1xceAka4+/ZwopdzmdnNwMPuPlA5DIeKMxEREZEYomlNERERkRii4kxEREQkhqg4ExEREYkhKs5EREREYoiKMxEREZEYouJMRGoMMyszs/UVfiZdoP84Mxt1Bc67w8yaXu7niIiAltIQkRrEzL539wYhnHcH0M3dD0T73CJS82jkTERqvGBk61kz+zz4aR+0P2lmDwfbD5jZFjPbaGZzgrZ0M5sftH0WLLaKmTUxs0Vmts7MplHhOYNm9vvgHOvNbJqZJYbwlUUkjqk4E5GapN4505p3Vjh2zN27E3m6wHOVvHcS0NXduwDjgrangHVB22PAzKD9z0Ceu3cl8hib1gBm1hG4E+gZPAS8DBh5Zb+iiNR0dS7cRUQkbpwMiqLKvFXhdUolxzcCb5jZfCKP/wLoBdwB4O65wYhZY6A3cHvQ/oGZHQ769wV+BayOPGWKevz4wHYRkYui4kxEaguvYvuMAUSKrkHAE2aWQ4XpykreW9lnGDDD3R+9nEBFpHbTtKaI1BZ3VnhdWfGAmSUAWe6+DJgIpAENgE8IpiWDh0EfcPdj57TfClwVfNRSYKiZZQTH0s2sTTV+JxGpgTRyJiI1ST0zW19h/yN3P7OcRoqZrSLyR+ld57wvEZgdTFkaMMXdj5jZk8D/mNlG4AQwOuj/FPCWma0FPgZ2Arj7FjObDCwKCr4SYDxQeKW/qIjUXFpKQ0RqPC11ISLxRNOaIiIiIjFEI2ciIiIiMUQjZyIiIiIxRMWZiIiISAxRcSYiIiISQ1SciYiIiMQQFWciIiIiMUTFmYiIiEgM+X+ioCu9NovrpwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x504 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10,7))\n",
    "plt.ylabel(\"Duration\")\n",
    "plt.xlabel(\"Episode\")\n",
    "plt.plot(running_mean(time_steps, 50), color='green')\n",
    "plt.savefig(\"PG_score_plot1.pdf\", format=\"pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1221a3d68>]"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJztnXl8FeX1/z8nhBC2sAXCqgEBBal1iYDFrQpfl2rR/mpdWqWioLZa21rXftVWrbVqW2u/tH5RW/EraF0LVdQqWi0W2cQFWQSsRCCYQEjYSQjP749zn85z5869d+Zuc2fmvF+vvGbu3EnmyST3+cxZnnNIKQVBEAQhepT4PQBBEATBH0QABEEQIooIgCAIQkQRARAEQYgoIgCCIAgRRQRAEAQhoogACIIgRBQRAEEQhIgiAiAIghBRSv0eQCoqKytVdXW138MQBEEIDEuXLt2ilOrt5tyiFoDq6mosWbLE72EIgiAEBiJa7/ZccQEJgiBEFBEAQRCEiCICIAiCEFFEAARBECKKCIAgCEJEEQEQBEGIKCIAgiAIESW8ArByJfDSS36PQhAEoWgp6oVgWfHVrwJffAFs3Aj07+/3aARBEIqOcFoABw7w5A8AAwYAt98OfPqpv2MSBEEoMsInAHv3Aj//Oe+fcgpv77gDmDLFvzEJgiAUIeETgJIS4KGHgDFjgGefBZYu5eNLl7JlIAiCUMzU1gIrVhTkUuETgLIy4P33gQULgB49gKOPBh5/HGhuBj780O/RCYIgpObuuzmGWQDCJwAA0K8fQGS9Pv543r77rj/jEQRBcMuePUDHjgW5VDgFwE51NdCrF7B4sd8jEQRBSM3evUB5eUEuFQ0BIAKOPVYEQBCE4kcsgDxw4onARx8Ba9b4PRJBCD+NjcDWrX6PIpgETQCI6HQiWk1Ea4noJof3OxDRX2LvLySi6lxc1xOXXgq0bw/ce2/BLy0IkaNXL6Cy0u9RBJMguYCIqB2AaQDOADASwIVENNJ22mUAtimlhgL4LYBfZXtdz/TtC1xzDfDII8CVVwJKFXwIgiAIaQmYBTAawFql1KdKqRYATwGYaDtnIoAZsf1nAZxKZKbpFIhf/AI46yzgf/+XawUJgpB79u3zewTBJmACMADA58brDbFjjucopfYDaAbQKwfX9kZ5OS8SA4AXXij45QUhEjz5pN8jCDZ79gTHBQTA6Une7l9xcw6fSDSViJYQ0ZKGhoasB5fAgAG8yOK++4C1a4G2Nj6+ejVw441AS0vurykIUaGxkeNtgjcaG4Grr+bJf+/eQFkAGwAMMl4PBLAp2TlEVAqgG4BGpx+mlJqulKpRStX07t07B8Nz4NFHOQYwbBgwaRIf++lPOUB8++35uaYgRIEtW/weQTC56y5g2jTgsccC5wJaDGAYEQ0mojIAFwCYYztnDoDYTItvAnhDKR+jsIMHA08/zfszZwI33AC89Ra/vv9+YNUq34bmC+vXA3OMP1lTk39jEYKN/t857DDeSv0td7Rrx9sdO4IlADGf/tUAXgWwEsDTSqmPiegOIvp67LRHAfQiorUAfgwgIVW04Jx2GvDJJ7x/33385PLnPwP79wPPPOPv2ArJ3r28UnriRP6wrl/PNZTuucfvkQlBpLmZt0OH8nbvXv/GEiQ6d+btzp0cRC9QDCAnDWGUUnMBzLUdu83Y3wvgvFxcK6cMGwZs28Zpod27A9/9LnDnnbxgLCqYq6O3bQPefpv3b74ZOOoo4KSTCvbPKIQAbQH068fbPXuATp38G09Q0AKgF88FxQIIPN27A089ZWUHHXEEWwB9+/I/8Xe+A+ze7e8Y88mGDdZ+QwOwcKH1+vTTgR/9qPBjEoKLtgC0AIT5s5NLtEiKAPhMr1h26hdfAJs3c4zg97/3d0z5ZJMRr29oABYtin//vfcKOx4h2IgAZIYOiWoBCFAaaLj44Q+Biy/mtKxFi4BDDrGayoSRjRut/YYGbkahA3gAx0QEwS1NTVx8sU8ffr1nj7/jCQo6/VxnUYkF4BOjRnEDmR49uILoqFHhjgls3Ah06cL7/+//seVz5JHW+8uXW8FyQUhHczPQrZvl0xYLwB2trbwVF1CR8aUvcQXRsGYzbNzIcQ+TL3/Z2m9pAQ49tLBjEoKLFgDt0xYBcIe2AMQFVGR8+cu8Wvjss4EZM9KfHyR27+an+8GD44+bLiCNmPKCG5qa4gVA/m/coQVAC6ZYAEWC7s35+uucJvqvf/k6nJyhFHD99ez3v/zy+Pd0AM9EgsGCG7QFoCcwsQDcYS9BIwJQJPSy1awbNy4cvYV//nPgD38ArrsOOPlk9v1rtAAQAXNjyzt+/WsWQUFIxe7d7P8XF5A3dAxAo+NyeUYEwA3z5wPPPWe5Rn7zm/j3H3wQGDnSKixXKJYv54J2mfDAA+zW0g1y+vQBamp4v29fzkZobOS1AJ06cfXUCRNyM24hvOhmJuIC8obdAiiQAORkJXDoGTeOtyNG8ES/a1f8+2+9xf0F/vUv4IQTCjOmAwc4QA14b27T0sKm+rHHAiXGM8CrrwIffwyUlcVbPgcfLP0TBHfoSpYVFfy/ZaYZC8nxSQDEAvDCiBFcM6e2lp+OL7+cfej6KbyQPQb++c/Mv1fnGturrfbs6SxgOqUP4LLZgpAMXcu+vJwfMN580+8RBQNxAQWEgw5iAXjhBS4r/f3vWwLw298CL77IxeWI8ruIyvTH69WXbkkmAMkw/zkPOyx+9bBm1Spg3jxv4xDCh9nP9tRTeTHljh3+jikI2C2AAtVPEgHwyqBBwPbtwIcf8utnnuFA1zHH8Ouzz+by0gCLgV3Zc4W5OGv1ao4/uI0H6EY7bpt2D7A1eKutTTxnxAhg/HjnOEhbG1cXlYkg/JiljIcP5799Pho7hQ27ABSoY64IgFcOOoi3r7wS7z+/4w7uN2xy7rnAmWfmZxyffAIMGcL7q1dz0bZhw4C6uvTf69UC+NOfuFuaxn4NM0X0448Tv/+ll7i66PXXu7ueEFxMC6B9e95KOZH0+NSJUATAKzoT6JNPOB6wbBmwZAlP9H/7W6IIvP66u0nZCwcO8PXPOIMbSXz4oVWwzk2+vhYAtxZAVRXwy19a6wXsv48Z+3BaJ6GtAgkIhpv9+/lvrQWgNJZjki8rOCxs3x5flbeAiAB4ZeRIa/+YY7hujnb/AMCzzyauGM51Bs3Gjex2OvxwLlZ3//3We++/n/77GxrYxOzZ0/01ibhkdklJfAxg1ixuZ3f88Zw55CRA9lWOQjjRKZ/aBaQFQCyA1Bx2GH9uOnQo+KVFALyizVoA+MY3Et/v0AG45BLg00+tKqIrVuR2DHoh2jHHWP75SZNYDJYtS//9mzfz5F/qMQu4XTteI2BaAI8+ytspU9gFtW5d4vdt28Zbe/qsEC50vSy7BSACkBr9efLyQJYjRAAy4cEH2eUzYkTycwYP5o5a3bvnXgDmz+csgaOO4hZyAJewPuqo9BbA6tXc9cteAM4t/frFWwC1tcD557PoDR3qHIjWArBwIccDhHBitwAkBuCNHj0KfkkRgEy45hp3ExkRl5N245bxwvz5wNix/AH7n//hCfiEE9gdtW4dZ9zceWf897S2cnrqYYexS2rMmMyuPXAg9w3eto1dUevXW8Xkhg4FPv88sXKqFgAgMUZiZ9067okqBA+xALLDh9aZIgD55rjj2BWUi3LSd9zBk/1777HPHQBGj+aWlmVlVh3/m28GbruN3TL6qezll630VICthUw4/HAOQH/lKywGra3xAqAUu79MTAFweq3Zs4d/xqWXZjY2wV+SCUAUgsAtLVYp50wpUOqnSVYCQEQ9ieg1IloT2ybYMER0JBEtIKKPiehDIjo/m2sGjnHj+J9jyRIulfvsszxJeo36t7UBt98OPP00v9YCYGKf1B95BJgzh/dXreLt2rUczD33XG/X14waxWPRPw+w0lEHDeJtXR2ng86axRlLn37K58ycye+/847zz9YfoCefzGxsgr9kGgSeOzf4K8wvvNB9Vl0ydLJE167Zj8cl2VoANwGYp5QaBmBe7LWd3QAuUUodDuB0AA8QUfcsrxscjj+eg6cnnMA+vvPO4wVTgwbxBOkWe3B17NjEc/r3j89SAvjJX39/ZSUHiq+4Ij6Y7QVdf0hz2GGW8HSP/Vmbm1kovv1t4KSTgDfe4HISWrTMyqMmjY2JxzZsCG8znmJn27ZEay4VdgvAbQzga19z7kERJJ5/nrfZFL/bvZtdy3qRaQHIVgAmAtA5jzMAnGM/QSn1iVJqTWx/E4B6AC5XIIWAXr3iWywCPCEC8emb6dBxhPnzuUVlsqeEt9/mJ/8rr2R3yosvWquEDznE+/jtDB9u7T/xBMcTdOE4LQDmys/583lrjjnZimC7ACjFQumUbSXkH51m7BZJA3V+iEnFgQPW/u7dnFxSXZ3TIaUi22qgVUqpOgBQStURUZ9UJxPRaABlABxyBf9zzlQAUwHgIL3qNuhMm8YlpC+4gAOca9cCixd7U/rly9mSqKlJnS/cqxdw2WX89fTTHDNYsIAtACe3kVfKyoBTTmERM8UAsATAqQDY5MneBGDrVuv31FaMUFi8LmDMJAZgToBhYNu2xNIpqTDXxviwTiatABDR6wD6Orz1Uy8XIqJ+AP4PwCSlVNK/ulJqOoDpAFBTU+OxznGRMmYM8Je/xB+7+mrr6dgNdXVcusHLYpHTT+fzv/Y1Xm147bXuvzcVzz7LfnrdP0DTpQsHsuxrEXbt4qdCIp4c3AjAkCGSMho0MskCClu/AK8WgE7jBnwRgLQuIKXUeKXUKIev2QC+iE3seoKvd/oZRFQB4CUA/62UCkE7rRzQrRv7yt3W8q+v56YtXqio4Po727ezWXnFFZ6H6UiPHsD3vpeYtVBSwr+XWagO4PQ2fW6XLukFYNo0PsfeeEcobjJxAYVtdXiyDLdkmJ8FH9Kfs40BzAEwKbY/CcBs+wlEVAbgBQCPK6WeyfJ64aGign3zbp+AGhrcF28zue02fpJeurQwecbaDVRW5vx+167xTz0mjY38fVddxasizcwQr01vhNzh9t5nEgQOy+rwdu1461UAzM+CD7GubAXgHgATiGgNgAmx1yCiGiJ6JHbOtwCcCOC7RPR+7OtI5x8XIbp1463bWv6ZCkD79hxYKtQycy0AOiVUd1PTdO2a2gLo2ZOthV694lcVNzXlfqyCO9zm8eunebsFkOr7w2IBaAHw6gLS/+OzZvmS/pxVEFgptRXAqQ7HlwC4PLb/BIAnsrlOKKmo4O327VYT9lTU12cmAIVGC8CAAc4FrtwIAMACYLqSPv/cl6XyAtg1kcyiM9FPs7qbVToX0M6d8W7JAwfiS6wHES8WwJ49wLe+xftDhri7xzkm4Hc7wGgLYPv29Ofu28fneY0B+IH+vQYMYJHzIgAbN3KxOSBxUY1TkTmhMLitVb9jB7sZ9dNwOgGYNi0+ESKoDYNaWqx75MUCMCvneskcyiEiAH6hLQA3LiCvDVz8ZPNm3p58svP7qQRg3Tor79xsSg8kBpaFwuE2OLljR3wv23QxAPsTr5uHoWLE/H92apeajIULebtuHZdV8YFs1wEImeLGAlizhrN4To152aqq8j+ubLnrLl4jMGWK8/vJsoC2b+c4h10AevXiJ8k1a/IzXiE9bgVg5874BYrpLIDutoIAYRCARYs4aO6mrs+iRdxhUJdS8QGxAPzCtABqa3mRmJkR0NLC9UVmzwZ+8ANeKj9+vD9j9cL48cDddyf/ACSzALSLZ+hQ3moB6NyZ+wyIBVBYzMwfLy4gJwFIFgS2r2YPqgDocZ9yClsATj2znVixIvOy7DlCBMAvTAvgmmt4odjf/269f//9nLpZU8NPzc89F29eB5WuXfl3tldO1AJgLz0wdiyLgsQACotZf8mLC8iLBWBfBZxtNU2/0K1OdalzN+Xf29r4ocbnGkgiAH5RUcE+0iVLgFdf5WPNzfzPc+AAMH06MGECd/9avz6xyFtQ0cXk7IvSdNkBHQw7+mje/vjH7B/dvDlaNWX8xkzPzJcA2I978Z8XE9o9qVfGu1nbUFvL91UEIKK0a8e1dJ54wvqATZ7MlTV//3ue9C+5hM/zoVVc3jj/fO5hUG9bNK7T57Rf+LTTeBIaM4ZF4cCB5FVEhdxjCoBbF9DOnfFWakkJf7kVAP0kHTTWrmXh07XL3FSv1eXURQAiTLI//gMP8DZZJk3QqapK9Pdu28aTh1mmWi8o0lZBUCeIIDJjhrWfqQUAsBWQLAZgCkDfvsH9+65Zw3Eq/f+aSgBefplFUReCLGDlTydEAPxEB0r/+Mf44599xk/CPuUG552KCmcBSLbQS9+HefOAv/0tv2MTeAK79VbrdbYCkMwCMIWhf//guoDWreM4lS6BkUoA7r6bA+yvvcavfU7tFgHwk3Ni7RNOOslK8fzOd3hbWelLi7iCkKkA3HIL8PWv53dsQmLGVTIXkLnqVanENFCALbp0LqD//m/+GwfVAmhu5v9dNwKg1z5s2MDf48PqXxMRAD/59re5xs2IERwM/uAD/jAALAphpaIiMRU0lQD07m0FFIX8smtXYtlwJwtg9WqOTf3pTxybeewxFgEvFoA+/uMfswB4bZNaLOzbx5N/+/b80OZWAIpgZb8IgN/odNCBAzkn+NBDgX//24oDhJGKCn6qNCeWVAJQUsItJjVtbe6vpRTwu98Fd3IpNLNmAf/4B+/r1alOAqBbRT72GPebmDyZX9tTld3EAEpLeTFUY6P74ojFxN69XPJE97twIwC7dokACEmorg5Hzn8yzEJ4mlQCAACTJln7XipIvvMO8MMf8mI6IT06F79dO+Bf/+J9JxeQtuC++ILXq2jshQ3dWAClpdYCwKCt+FbKsgAADgS7EQBABECIKJkIwLXXAhdfzPvJ+gk48fTTvA1b56l8UVvLk1h9vVXIz8kC0H2fde0njX0hX6oYgLYMSks5iwYIngBocdT3yq0FABRFaRcRAKHw2AWgpYWf6lMJABEvjAO8NRFZvJi34gJyR20tuyF79rQmKycB0Os47MH8wYPjX5eWAo8/zjEuO6YFoIUjaAKg7422ANIJgFnuugiKO4oACIVHC8Ctt7LPVz9FpjOJO3fmrRcB0E+qy5dLOQk31NZaC5r0U21LC69QJ7LKN+v7asfuutTuumOPTfTv79/PriYitjoGDrRiC0HB3gWtvDy1tWmKg89rAAARAMEPtAC89BJP+gcfzK/t7gM7WgC8uIAaGqwP2iWXeBpmJHESgH37rHIlc+bwtr4+/d8LiO/ktn59/Hv798dnd/Xrl+hSKna0BeDWBWSKg8+rgAEpBy34gfmUaAYY000o+vvcWgC6kc7113OJaiklkRql+CldV2ItKeEJet8+a4W2dmHU1/MT+5QpLOKdO1sCbWIG7O3B+9bWeAHo25cF6OijgeOPBx58MHe/W75wsgDcCsChh+ZvXC4RARAKz7BhwE9+Apx+enyJa91HOBleXUA6o6Wykn920NwLhUYHZc1AZVkZC0CnTvxaC0BDA6fm3nij+59v/7vt3x9f+qOqimM2mzcDy5YFQwCysQCKoMVp1i4gIupJRK8R0ZrYNulvRUQVRLSRiP4n2+sKAaZdO+C++7jRjTkp61aCyfDqAtJ+6t69+Xu9xA6iiBYAc1Lu0wdYsMAq3awFYNs270UK7RaA3QVUVRUsF9COHdb/lFcL4JRT8js2l+QiBnATgHlKqWEA5sVeJ+NOAG/l4JpCWBg8GPjlL7mTWDq8uIDWrrUygEQA3KHdcaYFcP31vB5Aty/U5UmamhI7eqUjnQDoftDFzpYtXLKlooL7VQDeLIBvfYvrWhUBuXABTQRwcmx/BoB/AEiwC4noGABVAF4BUJOD6wph4aZUzwwGXlxAOq8csARgzx5+ki2R3AdHnCwAXeNex09KSniC27fPuwDY/272GEAR5MW74s9/BmbOjD/m1gLYvduqGloE5OKTUKWUqgOA2DYhl4+ISgD8GsD16X4YEU0loiVEtKQhWaqZEE0yyQICrCAlkPgUamapRB0nAdBWl873LylJ7N3gFicLwLyWaQEUcyFEXb7FJJ0FsHo1cPXVLIJBEwAiep2Iljt8TXR5ne8BmKuU+jzdiUqp6UqpGqVUTe8iWCghFBE6RvCzn7kvB1FRwVktTtbDk09yIE7XZo86Ti4gXdxN5/CXlFii6TWI+f778X83uwvoyCOt/SIIkCbFqad1Ogvg/POBadO43lHQBEApNV4pNcrhazaAL4ioHwDEtvUOP+I4AFcT0WcA7gdwCRHdk6PfQYgS+inx4Yfdna8nficB+POfebt2bW7GFnScLIBUAuDGAnj3Xav2/aOPAt/4hvWeXQC6drVcd/aqosVEY2PisXQWgFkOI2gCkIY5AHSlrkkAZttPUEp9Wyl1kFKqGsBPADyulHLp+BUEgw8+4PzpP/wB+NWv0lsCOn3RSQB0/XkvxeXCjJMFoF1AelJTypsAjBkTn+qrF5QBiTEAgEtGHHus+yY0ftDYyBaK+b+UzgIwXVohE4B7AEwgojUAJsReg4hqiOiRHPx8QbDo04cXH33yCQeP01kCV17JW6cYgK4PFLVY09ChwMiRicedLIDS0vgJq7U18xiAHXsMALAya9z01fWLrVv5/1A/XADxFsD+/YkF8IpUALLOAlJKbQVwqsPxJQAudzj+GIDHsr2uEGFOOMHaT9ZRqUMH4NJLgeuu49f6w/rKK7z24LzzrMBm1AQgWU0kJwEA2ArQ+eutrZnHAMzr6CqhTo1+0mXS+E1jY+IaCNMCANiCMX83M/NMr7QuAmQlsBA8jjrK2nfK4tE12quqrCcvbQH87Ge8Nc33qAlAMpxcQAD74/U9ammx7rlTNowb/v1vYPjw5AKga+orVZzZQI2N3MPYxLQAAB6/WRrDFIAi6AOgkYRoIXi0bw/Mncv7W7Ykvm8v0Qsk1qkxm8uLADDJLAAzINvaygHhsrL4++sF7XpzigEA8U/RxUhjY+JTvJMAmJhCJgIgCFlyxhlcRXTLFl6lqksVANaHz/S12gXAbHwuAsCksgDMc3btci785pa6Ot46xQAAd83V/WTbtsT4h37CTzZ2MyYgAiAIOaCyEnjqKQ4aTptmHbdXaASSC8CQIVZzk6jj1gLIVgB0vZ9UMQCgeAUg1e+vxz5lCrBpk3XcXLxYROubRACE4FJZaT21rlhhHdcBS1MAdA8CjbYYRo3iD6pS+RtnMeHU31fjRgB09zavAqArvXbsGG8BBE0AWluBtjYrqWDFCl5QqNFW55tvAt/7nnXc7JxWRFlAIgBCcKmstPb15AU4WwDt2yd2qwKAI47gCS0qJSFSldFI5gIy71umFsCKFZw+2a+fJQDJYgB6gkzVWau11R/R1mnEeowjRgAXXGC9b/7PmYkGTquHiwARACG4mL5UM7XRSQAAayLTE1xJCXD44byvF4V5pa0t3voodlIV0ktmAZguCx0DMHPg3dClC6dOmgKQaQxg927+G955p7cx5AItSsl+f/N/TgtqS0vRBrRFAITgctVVwDXXJPaSTSYA+qnVLDeg2x9m2jR+yhQWkaDEEdxYAPZJ2Wz03tqamQtIowVg5kwWzkxcQLoshZsS4rnGbgHYMf/ntKDqp/8rrgA++ih/Y8sAEQAhuAwbxl2jJk8GPv/c+sClswCGDuVt+/YsHkDmAqDrCdkbnhcrqQTAqSMYEC8A2WYBVVayK+g73+HXTnn+6QTAXJRWaLxYAHp8+p6PHs0xpyJCBEAIPoMGsT9Yuxac0kABa9LSqzivvpqfSIlYQLxi+nVT+auLiUxcQEOGxJ+TjQD06hVfTM3JckoXA/CzdpO+thcXkL7n2WRO5QkRACH42J/ik1kAV1zB21tuAf76V+DWW3my69HD6h/sBXMRWlAKymUSBNZuMsASAK8xAE3Pnhw30axfn3iO/rslEwBTxMz1H4VAj8mLC0gEQBDyiBYA/RTvlAYKAJdcwoHHoUOBiROtxTsVFZllaZhun6BYAG5cQHYLoKzMWumaaRqoxl5D57PPEs/Rq2yTLdAzxVZ3KisUYgEIQpHh1gIAnBvPd+0an6ftFlMAgmIBZOICAviennVW9gJgL6HgVE9I13DSLj075r3OxHLLBi8WgB6nCIAg5JFu3fjD5UYAnKioyEwAzLUDQREAp9pJmpYWnnidRBJgYdi+neMtubAAfvELYMEC5+v07g288w4wf36im8e814UOvnuxAHTZbC0ATutQfEYEQAg+RMCAAdbSe/0hzbcAmE+fQXEBLV1q7dsnVl2mOVkFzvbtrUktFxbA9ddbKbl2+vUD5s3j0t9z5sS/Z1oxhRaAdBaAmda6dy+PVbvdxAIQhDxRUWF90LZu5cnKbVvBTATg6quByy6zXgfFAli40No3g7EAWwDJ+isA/J62erIJAmucXE0aU7ztbh7zXhd6BXe6NFA769eLC0gQ8k6nTtYHrb6eXQhua8mnCgK/+y5w443xxxoa4ovPAcGwALZsAWprrRIa9q5V2gJIRvv2ltWQ6WTmtomMuS7Dvh6gGFxAbuv5/PvfIgCCkHc6dbI+nA0N3ioupgoCH3cccO+98UXUnJqC62tv2gRMnVqchcx0zn11NW+9CoBTr2CvtG/PpbxnzEh93pFHWvupBMAvC8Cte/Gss4DXXuOHkUz7J+QREQAhHHTubE0M9fXeaq5XVPBTmt0lYqJ930BiKmVJiTUx3Hgj9yl+4QX31y8UerLU4mgXgHQuIFMcsilpPHcup+Sm4oknOAgMOAtAWRmPxw8LoLw8vsNXOt54g/8/i7C7WVYCQEQ9ieg1IloT2zrad0R0EBH9nYhWEtEKIqrO5rqCkIDpAvJqAehS0atWASeeCLz/fuI5pgDYUylN60NPkqnSLf1C/w7aBWQvpZDOAjD93vluatK9O/CVr3BGkt29plcid+/uTxDYjf/fvt6hCN0/QPYWwE0A5imlhgGYF3vtxOMA7lNKjQAwGkBAKmcJgSFbCwDgbJN//pN7DtszZEy3j90C6NQpMThYjEFhNxZAKgEw++AWqqmJU4P43bv5PneOCny6AAAZT0lEQVTrVngX0O7d6f3/W7aw7/+VV6xjIRWAiQC0M28GgHPsJxDRSAClSqnXAEAptVMpVYSfDiHQaAtgzx6eoDOxAJYssY5p94PGyQV0663ArFk8IegJX3/Qi9ECSCcAO3ak9u3rpi5A4Sa0dAJQjBZAr178PzV+vHUspAJQpZSqA4DY1umxaziAJiJ6noiWEdF9RJRkpQlARFOJaAkRLWmQXq2CW7QbRpcG8GIBaHP91Vc5/7xjR+C55+LPcRKAyy8HLrww3gLQPvS//CXzCqP5wu4CsgtAXR3//skwBaBQJBMA7QIqRgtA066dZVEFVQCI6HUiWu7wNdHlNUoBnADgJwCOBTAEwHeTnayUmq6UqlFK1fQuot6ZQpHTuTOvUF2+nF+bFSzTMW4cP03u2gXU1ACHHMLpku+9Z53jJAD6adm0AHTjjw8+4AyQYqKpicVKP8HaBWDTpmAIgC5G16VL4S0ttzEAjc78CaoAKKXGK6VGOXzNBvAFEfUDgNjWybe/AcAypdSnSqn9AP4K4Ohc/hKC8J8PpZ60Dz3U/feWl3NPgZISnrR79+Y4wjHHWOekEgDTAjADlpmUmM4nTU381KxXq5oCMGcOW0+mn99OvgO/TpSXJwaBt2/n1F2n9/KNFwsACL4ApGEOgEmx/UkAZjucsxhADyLSj/OnAAhQDz0hEOgP2LJlPDGnmsicuP9+/nBPncoTXW1t/PtaAPbuBd5+m0177e7p1MkSBXNC8jqGfLNtW7wA6CygPXu4OiqQ2gLQqY+nnpq/Mdrp2DHRAtixg33sHTsWXgC8WgBaLIpUABz6sXniHgBPE9FlAGoBnAcARFQD4Eql1OVKqTYi+gmAeUREAJYCeDjL6wpCPPpDuWwZMHy495zrkhKr5HGfPolP71oAzjwTePPN+PeqqjiFFIjP/klWVM0vmpp4Ja72S2sLwMxwSiUAAE+ATm0c84WTC2j7dhaAsrLCL7gLmQWQ1V9SKbUVQMLjgFJqCYDLjdevATgim2sJQkq0AKxfz/nj2eAUe9qyBXjqqcTJH+By1Js2ceqofiLt27fwterT0djIY7W7gEz3llN5ZpNCr2YtL09MqdUCQFT8FkCRC4CsBBbCgfkBO+yw7H6W6et+4w3gv/6LYwI33+x8/sCBPJnW1/MEMXo0xxQaGgrfsSoVX3zBv5tdALQFUFUFjBnjz9iSYbcA2trY3Wa6gJQq3HhCZgGIAAjhwHwq8xIAdsK0AE4+mSfG9ev565ZbEs8fMIC3GzfyhNSxI39PW1vhG5bYaWsDfvQjYPp0FqSqqngBWLnSyld/6SVvk1sh0IHeFSt4vDrWogXgwIH4Fc1tbcDs2fkTBbEABKEIyYcATJ7MboaqKn66VwoYNSrxfLMjmX5CrKriY7pHgV88/DDwwAPcD3n//ngBaG0FLrrImkDt5QuKgfJyFqnDDwduuAEYO5aPV1Q49w6+917gnHOAF1/Mz3j0IjS3iAAIQgHQkzBR8iYjbjn+eOD554GHHuLXejIHOMD8yCPAzJmJ1/7sM+sJUVezNOvv+8G6dfGv7S4gM9vJbanmQmLGHGbMsILt2gIA4gVA13Gyl/fu3x+46qrsxtLayhaGuIAEociorOQsl7Vrs/+wEQHnnmtly5gCMGwYN4K56CLrWJ8+LAwzZ1ouoOHDORDsFDQuJPaVslVV1u+1d298BpAuiVFMmJOtGQswBcA8rn9fexZYXZ0l6JmSrh2kEyIAglAgunXztgLYLWZQ2GmSJAKuuQZYvBj49FOemIi4neGiRbkfjxeamuLHb1oAq1fHn+ulxHGhcGqyDiS3AHRtIFP4chUPSNcO0gkRAEEIOLqP7TkJtQ4tzj3X2tcTRL9+qZuwF4Lm5nhRNGMANyUr3ltEJEs77drVvQDo8hzZcvvtvA2RBVDAFR2CEFCOOQb44x/j3T52BgzgAPHy5daCsh49OGe9rc2/RWFNTRzc/egjrlxaWZmYmXT22clbYvpNMgEoK3MWAL32whQAr/2enWht5UwqIDMLINMOanlGLABBSAcRcOWV6X3kP/oRb3V+vc6qKXTFSpPmZi7/MGoUcPfd/LvYV/LOmeN/rCIZXbsmHhszhgv22bOA9u2zFrXlWgBMS04sAEEQErj0UhaJceP4tc6qaWy03EiFpqkpcXWvKQD33FPY8Xile/fEY3/9K/8OdgvALB9vCkAurJt6o86ltvDcIAIgCBGBCPjmN63XWgDMUguFRlsAJmbXrxtvLOx4vOJUmsIsww1YWUDmJG3e81xYAObP9tKEpsgFQFxAgpAvtACMGeNPXaDx49ktYp9EdVaMbgxTzDhZANoFY7cA9CRdUZF7F5C2Lk49lQsCuuXoo7nHRDGusYAIgCDkD/NDP9upUnqemTePt/anz759gQkTErueFSN28erSxUpXTSYAw4fHWwC5dAE9/bS3IPBpp3F6cKpeyz4iLiBByBdmaQXdO6BQmEXo7CWTS0uBv/+9sOPJFLsFYGbTJBOAESPiS0HkygVUWupskQQYsQAEIV+YFkChXUD6Cfjww4Hvf7+w184lThaApnNnjrvoib++ngO02gLQwpcrAejduzgXy2VBuH4bQSgmzGyRzZsLe23ts77llqLNQXeFPQ3UXNVbVsa9H15+mV/rSVo3tdH33BQAex9kt+guZCFDBEAQ8smGDbxIrNAWgBYAP/r45hL7Ajq7kE6cyAXgNm2yJmm7AJgZPJk2kNm/v2j9+NkgMQBByCcDBgDV1cktgMWL2a+cbQVTO3rSc+puFlSeeirRItClv+vqrEqsfftaxwCrQijAbiGnxWXpaG0tbCvMAhG+30gQio2+fbmmvROjR/M21w1MtAUQJgE4//zEYzpG0NxsVWI1LYDWVuDDDzkes21bdhZACAUgaxcQEfUkoteIaE1s65jwSkT3EtHHRLSSiB6MNYgXhPDTq1fhO4NpAQhCrn86nnoqeYMXnZXT1GQ14+ndm4PDdXXcP2DfPqtPdKYC0NoaShdQLmIANwGYp5QaBmBe7HUcRPQVAOPAjeFHATgWwEk5uLYgFD8dOyamYgL57Rfc3MzukEKnn+aD888HvvY15/e0BdDUZLmASktZdOvrLVeYdrE5/R3cIBZAUiYCmBHbnwHAqWauAlAOoAxABwDtAfiwNFIQfMDe2FxjLlDKRaqiye7dRVt+IKdoC6C5Ob5he8+e7PLRPQT0mgwJAseRCwGoUkrVAUBsm5B2oJRaAOBNAHWxr1eVUkmcooIQMjp2ZDeE/YnfXK26fn1ur7lrl7eqlUFFB3RNCwDgCb+x0ZrwsxWAkAaBXQkAEb1ORMsdvia6/P6hAEYAGAhgAIBTiOjEJOdOJaIlRLSkwazuJwhBRRcEszcmMQXg4Ydze81du6JhAbRrx6mfZhAYSLQAdDXWbFxAUbUAlFLjlVKjHL5mA/iCiPoBQGxb7/AjzgXwrlJqp1JqJ4CXAYxNcq3pSqkapVRN7zBlMAjRRQuAffIxBeD3v89t1dCoCADAbiAzCAyIBeCSXLiA5gCYFNufBMCp6lUtgJOIqJSI2oMDwOICEqJBMgHQDdnvuou3q1cDc+dy8/K2tuyuGZUYAMCB4MZGtrDsLqBcxgBEABy5B8AEIloDYELsNYiohogeiZ3zLIB1AD4C8AGAD5RSf8vBtQWh+ElnARx3HG9XrwYuvhi46irgmWeyu2ZUYgAAWwB6oZ1pATQ3W4F2LQCZuoBCmgaataQppbYCONXh+BIAl8f22wBcke21BCGQOPWuBSwBOOYYfrpcvdqyCjZsyO6aUXMBvfce75sWAMAlIkpLrWCxWABxSC0gQcg3ySyApiaeVCoquMftu+9a7+3cmd01o+QCGjAA2LiR900LAGAh7dQpuQi7JcpBYEEQsiCZAOindCJgyBDgrbes97JtYhIlC2DoUGtfT/Q66+fll/lYsr+BWyQILAhCRqQTAAA4+OD4dQK5EICoxAAOOcTa17/zSSdZi8Q6duTJu7RULAAbIgCCkG/szcs1pgBUV1vHu3fPTgDa2vhaUbYAOnYEbr2V93UPgI4dJQ3UhgiAIOQbbQHYJ59kAjB4cHYxAH2dqAjAkCHWvmn16HVE+l4mq8nkBgkCC4KQEaYLSCluEL9gQbwA9O/P2+pq7uC1YwdXEH3kEfelorduBc45B1i3jl9HRQDMSd9009gFoLxcXEA2RAAEId+YArB0KU/SX/lKvAAcdRQwbpzV9GTHDmDqVGDKFK5nn4o1azij6He/Y3H59a/5eFRiAAAvoKusjG+so7uhiQsoKSIAgpBvTAHQef4AlyrWAtClCzB/PjBmjCUAa9fye59+6vxzb7kFeP11boJ+zjlW6eP/+z/eRsUCAIAzzuAeCGb/A3spmUxdQAcO8JdYAIIgeMYMApvB3dpa50laC4DOClq1KvGc1lbgl78EJkzg1wsW8KInk5qa7MceZOwCkKkLSFsQIbQAwvcbCUKxYQaBzeBuW1tyAdi82Spv4CQA9ibzX/pSfNvJ++6LDyxHEX3fNZm6gLQAiAUgCIJnSku5bPHevYnZPU4CUGL7WK5aFd/YHEhsMl9aynEAzeDBmY83TDzyCLBwIe9n6gJqbeVtCC0AEQBBKAS6K5gbAdBZPA89BJx9NrBoEQeJly+3zqmri/+ebdvi+w2IADCXXQaMHs374gJKQARAEApBp048+bsRgDvv5OyfyZM5OKyZNYsrXALOAtDSYr0WAUhEXEAJiAAIQiGorAS2bGEBMH3TTqmao0YB06fzhKMDyAAHfU84gfftLiC7BdCjR+7GHhbKyzn11u26Co24gARByIrevTlNcefO+Mk5XaqmXSA++oi3ZrnoPn2sp9Q77/Q+wUUFIg6e33CDt+8TC0AQhKwwBaBrV6CsjI/bM1XsmBaAZtEidgeNGwdccEH8hNahQ+7GHDa2b+ftQw95+z6xAARByAotADt2sF//rbeAI44Ajjwy9fc5uYhmzWJXxm9+Azz5ZHy6pxYWIZFf/Yq3XtZH7NrFrjsglBZA+CRNEIqR3r25Vk9zMwvA2LHABx+k/z4nAdBZQrrmvelSEgFITnU1rxjWK6bdcPjhwPr1vC8WgCAIGdG7N/vma2vjM3vS4eQCshd7M0VCXECpqajwVmpbT/6ACIAdIjqPiD4mogNElNSuIqLTiWg1Ea0lopuyuaYgBBJdlsCrAKSyAPR7pkiIBZCarl2tWIBXQugCytYCWA7gGwDeTnYCEbUDMA3AGQBGAriQiEZmeV1BCBa6MiWQnQXQp4+V7+8kAGIBpEbXWcoEsQDiUUqtVEqtTnPaaABrlVKfKqVaADwFYGI21xWEwHHQQdb+oEHuv89uAejvLSuzJiSxANxTUcGB3bY2798rFkBGDADwufF6Q+yYIEQHUwDMDlbpsFsA/frx1hQGsQDc07UrbzPpuBZCCyDtb0RErwPo6/DWT5VSs11cgxyOJV2pQkRTAUwFgIPMD40gBBnzydxsYp4OuwWg692bC8jEAnCPFoAdO4Bu3bx9bxQFQCk1PstrbABg2rwDAWxKci6UUtMBTAeAmpoaWdIohI9MLYAVK4CHH+Z9sQAyo6KCt27jAB06WCU2xAWUEYsBDCOiwURUBuACAHMKcF1BKC7003tfJ4M6CXqir6wERoywcv7NycgsHy0WQGpMC8AJpYDrrrN6K5j31l6mOwRkmwZ6LhFtAHAcgJeI6NXY8f5ENBcAlFL7AVwN4FUAKwE8rZT6OLthC0IAee89XgFMTl7RJGgB0N+jBcCs/GkiApAaLQDJUkHr6niF9WmnsRjs2QOcdBKvHvZiuQWErJxaSqkXALzgcHwTgDON13MBzM3mWoIQeAYN8pYBBCTWCtICYFb+NBEXUGq03//uuzkbaGKShMTPP7dKbp92GnDzzYUZX4EJX1RDEMKE9u/rpiY9e/JWLIDMGD6ct2++yV/2yqmmsA6IJSs6rcYOCSIAglDMdOvGDd9HjeLXYgFkR7rJ3ElYQywA4YtqCELYGDvWWj2cTgDEAkiP2SLSjtN9FQEQBKEoEAsge2bP5owqILULSCMCIAhCUaAFQGez2BELID19+wLf/S6wdy8Hgk1EAARBKFratwd+9zvgnXec3xcBcIeuztrQEH88YjEACQILQtD4wQ+SvxfCxUp5wRSAwYOt42IBCIIghJxkFoCTAIRYVMP7mwlClHBqHCMkR5fj0Iu9NKYAnHgib3VT+BAiLiBBCAOrVgGffeb3KILDgAFAu3bA9ddzZdULL+TjWgDWrOGey9OnA+PG+TfOPCMWgCCEgUGDgBNO8HsUwaG0lHs0NDUBF11kHddB4A4dOOPqxhvFBSQIghA6nNZMaAsgIuspRAAEQYgmjY2Jx0QABEEQIsDkybwdONA6pgUgIuspRAAEQYgmd90FnHtufH8GMwYQAUQABEGIJu3aAf36Abt3W8f27eMAcYgDvybR+C0FQRCc6NQpUQAi8vQPiAAIghBlOnXito8HDvDrffsi4/8HRAAEQYgyegX13r28bWkRC8AtRHQeEX1MRAeIqCbJOYOI6E0iWhk799psrikIgpAztABoN5C4gDyxHMA3ALyd4pz9AK5TSo0AMBbA94loZJbXFQRByJ7OnXkbUQHIqhaQUmolAJCZRpV4Th2Autj+DiJaCWAAgBXZXFsQBCFrnCyACMUACloMjoiqARwFYGEhrysIguCIFoBnngHGjIlcDCCtABDR6wD6Orz1U6XUbLcXIqIuAJ4D8EOl1PYU500FMBUADjroILc/XhAEwTtaAG67jbdf/aoIgIlSany2FyGi9uDJf6ZS6vk015sOYDoA1NTUqFTnCoIgZIW9j8Jnn7ElEBHyngZKHCB4FMBKpdRv8n09QRAE19gFYP16YNgwf8biA9mmgZ5LRBsAHAfgJSJ6NXa8PxHNjZ02DsDFAE4hovdjX2dmNWpBEIRcYBeAAweA4cP9GYsPZJsF9AKAFxyObwJwZmx/PoDkaUKCIAh+odNATSIkALISWBCE6FJRkXhsyJDCj8MnRAAEQYguXbokHuvZs/Dj8AkRAEEQoovTItaIlIIGRAAEQRAsSgu6NtZ3RAAEQYg27dpZ+xFaBAaIAAiCEHXMp/7ycv/G4QMiAIIgRBsRAEEQhIhiuoBEAARBECKEaQEcfbR/4/CBaIW8BUEQ7GgBmD4duOgif8dSYMQCEAQh2uiFX2ee6VwaIsSIBSAIQrR58UVg1iygf3+/R1JwRAAEQYg2hxwC3Hqr36PwBXEBCYIgRBQRAEEQhIgiAiAIghBRRAAEQRAiigiAIAhCRBEBEARBiCgiAIIgCBFFBEAQBCGikFLK7zEkhYgaAKzP8NsrAWzJ4XCCjNyLeOR+xCP3wyIM9+JgpVRvNycWtQBkAxEtUUrV+D2OYkDuRTxyP+KR+2ERtXshLiBBEISIIgIgCIIQUcIsANP9HkARIfciHrkf8cj9sIjUvQhtDEAQBEFITZgtAEEQBCEFoRMAIjqdiFYT0Voiusnv8RQCIvoTEdUT0XLjWE8ieo2I1sS2PWLHiYgejN2fD4koVE1QiWgQEb1JRCuJ6GMiujZ2PKr3o5yIFhHRB7H78fPY8cFEtDB2P/5CRGWx4x1ir9fG3q/2c/z5gIjaEdEyInox9jqy9yJUAkBE7QBMA3AGgJEALiSikf6OqiA8BuB027GbAMxTSg0DMC/2GuB7Myz2NRXAHws0xkKxH8B1SqkRAMYC+H7sfyCq92MfgFOUUl8GcCSA04loLIBfAfht7H5sA3BZ7PzLAGxTSg0F8NvYeWHjWgArjdfRvRdKqdB8ATgOwKvG65sB3Oz3uAr0u1cDWG68Xg2gX2y/H4DVsf3/BXCh03lh/AIwG8AEuR8KADoBeA/AGPBip9LY8f98bgC8CuC42H5p7Dzye+w5vAcDwQ8ApwB4EQBF9V4opcJlAQAYAOBz4/WG2LEoUqWUqgOA2LZP7Hhk7lHMZD8KwEJE+H7EXB7vA6gH8BqAdQCalFL7Y6eYv/N/7kfs/WYAvQo74rzyAIAbAByIve6F6N6L0AkAORyTNKd4InGPiKgLgOcA/FAptT3VqQ7HQnU/lFJtSqkjwU+/owGMcDottg3t/SCiswDUK6WWmocdTg39vdCETQA2ABhkvB4IYJNPY/GbL4ioHwDEtvWx46G/R0TUHjz5z1RKPR87HNn7oVFKNQH4Bzg20p2ISmNvmb/zf+5H7P1uABoLO9K8MQ7A14noMwBPgd1ADyCa9wJA+ARgMYBhsah+GYALAMzxeUx+MQfApNj+JLAvXB+/JJb9MhZAs3aNhAEiIgCPAliplPqN8VZU70dvIuoe2+8IYDw4APomgG/GTrPfD32fvgngDRVzggcdpdTNSqmBSqlq8NzwhlLq24jgvfgPfgchcv0F4EwAn4D9nD/1ezwF+p2fBFAHoBX81HIZ2Fc5D8Ca2LZn7FwCZ0qtA/ARgBq/x5/je3E82Ez/EMD7sa8zI3w/jgCwLHY/lgO4LXZ8CIBFANYCeAZAh9jx8tjrtbH3h/j9O+TpvpwM4MWo3wtZCSwIghBRwuYCEgRBEFwiAiAIghBRRAAEQRAiigiAIAhCRBEBEARBiCgiAIIgCBFFBEAQBCGiiAAIgiBElP8PAHrqdg8Qg0gAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.close()\n",
    "plt.plot(running_mean(losses, N=50), color='red')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Actor-Critic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 439,
   "metadata": {},
   "outputs": [],
   "source": [
    "l1 = 4\n",
    "l2 = 150\n",
    "l3 = 2\n",
    "\n",
    "actor_model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(l1, l2),\n",
    "    torch.nn.LeakyReLU(),\n",
    "    torch.nn.Linear(l2, l3),\n",
    "    torch.nn.Softmax()\n",
    ")\n",
    "\n",
    "critic_model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(l1, l2),\n",
    "    torch.nn.LeakyReLU(),\n",
    "    torch.nn.Linear(l2, 1),\n",
    "    torch.nn.LeakyReLU(),\n",
    ")\n",
    "\n",
    "lr_actor = 0.0009 #0.0009\n",
    "lr_critic = 0.001 #0.001\n",
    "optimizer_actor = torch.optim.Adam(actor_model.parameters(), lr=lr_actor)\n",
    "optimizer_critic = torch.optim.Adam(critic_model.parameters(), lr=lr_critic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 440,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor(1.00000e-03 *\n",
       "       [ 7.1524])"
      ]
     },
     "execution_count": 440,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "observation = env.reset()\n",
    "#state, reward, done, info = env.step(1)\n",
    "#observation.shape # (4,)\n",
    "critic_model(torch.from_numpy(observation).float())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 441,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_fn_actor(pred, delta, discount): \n",
    "    return -1 * discount * delta * torch.log(pred)\n",
    "\n",
    "'''def loss_fn_critic(pred, delta): \n",
    "    return -1 * delta * pred'''\n",
    "\n",
    "loss_fn_critic = torch.nn.MSELoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 442,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/brandonbrown/anaconda3/envs/deeprl/lib/python3.6/site-packages/torch/nn/modules/container.py:91: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  input = module(input)\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "Trying to backward through the graph a second time, but the buffers have already been freed. Specify retain_graph=True when calling backward the first time.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-442-3baf9c919805>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     36\u001b[0m         \u001b[0moptimizer_critic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m         \u001b[0mloss_actor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 38\u001b[0;31m         \u001b[0mloss_critic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     39\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m         \u001b[0moptimizer_actor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/deeprl/lib/python3.6/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m     91\u001b[0m                 \u001b[0mproducts\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m         \"\"\"\n\u001b[0;32m---> 93\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/deeprl/lib/python3.6/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m     87\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[1;32m     88\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 89\u001b[0;31m         allow_unreachable=True)  # allow_unreachable flag\n\u001b[0m\u001b[1;32m     90\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: Trying to backward through the graph a second time, but the buffers have already been freed. Specify retain_graph=True when calling backward the first time."
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "MAX_DUR = 250\n",
    "MAX_EPISODES = 750\n",
    "gamma = 0.99\n",
    "losses_actor = []\n",
    "losses_critic = []\n",
    "time_steps = []\n",
    "for episode in range(MAX_EPISODES):\n",
    "    state1 = env.reset()\n",
    "    done = False\n",
    "    t = 0\n",
    "    discount = 1\n",
    "    while not done: #while in episode\n",
    "        state1_th = torch.from_numpy(state1).float() #convert to PyTorch tensor\n",
    "        action_pred = actor_model(state1_th)\n",
    "        action = np.random.choice(np.array([0,1]), p=action_pred.data.numpy())\n",
    "        \n",
    "        state2, reward, done, info = env.step(action) #Take action, observe new state and reward\n",
    "        psuedoreward = t/MAX_DUR if not done else -1\n",
    "        psuedoreward = torch.tensor(psuedoreward).float()\n",
    "        state2_th = torch.from_numpy(state2).float()\n",
    "        critic_pred1 = critic_model(state1_th) #State-value prediction for original state\n",
    "        critic_pred2 = critic_model(state2_th) #State-value prediction for new state\n",
    "\n",
    "        delta = psuedoreward + (gamma * reward * critic_pred2) - critic_pred1\n",
    "        delta = delta.detach() #Need to detach because we're using critic_pred2 in 2 places\n",
    "        \n",
    "        #Training\n",
    "        critic_target = psuedoreward if done else psuedoreward + gamma * critic_pred2\n",
    "        loss_actor = loss_fn_actor(action_pred[action], delta, discount)\n",
    "        loss_critic = loss_fn_critic(critic_pred1, psuedoreward)\n",
    "        losses_actor.append(loss_actor)\n",
    "        losses_critic.append(loss_critic)\n",
    "        \n",
    "        optimizer_actor.zero_grad()\n",
    "        optimizer_critic.zero_grad()\n",
    "        loss_actor.backward()\n",
    "        loss_critic.backward()\n",
    "        \n",
    "        optimizer_actor.step()\n",
    "        optimizer_critic.step()\n",
    "        \n",
    "        #End Training\n",
    "        discount = gamma * discount\n",
    "        state1 = state2\n",
    "        t += 1\n",
    "        if t > MAX_DUR:\n",
    "            break;\n",
    "    time_steps.append(t+1)\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10,7))\n",
    "plt.ylabel(\"Duration\")\n",
    "plt.xlabel(\"Episode\")\n",
    "plt.plot(running_mean(time_steps, 50), color='green')\n",
    "plt.savefig(\"/Users/brandonbrown/Dropbox/DeepReinforcementLearning/media/Ch 5/PGac_score_plot1.pdf\", format=\"pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 436,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x136e823c8>]"
      ]
     },
     "execution_count": 436,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJztnXmcHNV173+nZ9W+jgRaR0IsllgECLGYzZaNAQMCzCKZ2MQmj9h+OHHIewnEz8TGMQ52EmzHxIYErwmIzRiBBRiDMbtAgIQQQjDahUAa7dJImq3v++Pe23X7dlX1re7qrprp8/185lPVt25VnemuOnXq3HPPISEEGIZhmNogk7QADMMwTPVgpc8wDFNDsNJnGIapIVjpMwzD1BCs9BmGYWoIVvoMwzA1BCt9hmGYGoKVPsMwTA3hpPSJ6FwiWkVEbUR0g8/2JiK6V21fTEStqr2BiH5JRMuJaCUR3Riv+AzDMEwU6ot1IKI6ALcD+CSATQBeJaKFQoi3jW7XANgphJhGRPMA3ArgSgCXA2gSQhxDRAMBvE1E9wgh1gWdb/To0aK1tbXkf4hhGKYWee2117YJIVqK9Suq9AHMBtAmhFgDAES0AMBcAKbSnwvgm2r9AQA/JiICIAAMIqJ6AAMAdAHYE3ay1tZWLFmyxEEshmEYRkNE6136ubh3xgPYaHzepNp8+wghegDsBjAK8gHQAeADABsA/IsQYoePsNcS0RIiWtLe3u4iN8MwDFMCLkqffNrsLG1BfWYD6AUwDsAUAH9LRFMLOgpxpxBilhBiVktL0bcThmEYpkRclP4mABONzxMAbA7qo1w5wwDsAPBZAI8LIbqFEFsBvABgVrlCMwzDMKXhovRfBXA4EU0hokYA8wAstPosBHC1Wr8MwNNC5mzeAODjJBkE4BQA78QjOsMwDBOVokpf+eivA/AEgJUA7hNCrCCim4noItXtLgCjiKgNwPUAdFjn7QAGA3gL8uHxcyHEmzH/DwzDMIwjlLYiKrNmzRIcvcMwDBMNInpNCFHUfc4zchmGYWqI/qX0D24FNj0MvDAfyPYkLQ3DMEzqcJmc1Xf4zVhvff0C4LPpcl0xDMMkTf+y9BmGYZhQ+o/S37k0aQkYhmFSDyt9hmGYGqL/KP09PnO+9q6uvhwMwzAppv8o/dEfLWx7ZFr15WAYhkkx/Ufpj78gaQkYhmFST/9R+uSX6JNhGIYx6T9KHwCu6AAufj9pKRiGYVJL/5qcVT9Q/jEMwzC+9C9Ln2EYhgmFlT7DMEwN0T+V/qQrvHWRTU4OhmGYlNE/lf7p93rr796enBwMwzApo38qfZOO9UlLwDAMkxr6v9JfvyBpCRiGYVJD/1f6Bzhun2EYRtP/lT4AdO1MWgKGYZhUUBtKv/dg0hIwDMOkgv6r9Kff6K0vOjY5ORiGYVKEk9InonOJaBURtRHRDT7bm4joXrV9MRG1qvariGip8Zclopnx/gsBzLzFW+/cVpVTMgzDpJ2iSp+I6gDcDuA8ANMBzCei6Va3awDsFEJMA3AbgFsBQAjxP0KImUKImQA+B2CdEIJLXDEMwySEi6U/G0CbEGKNEKILwAIAc60+cwH8Uq0/AGAOUUGu4/kA7ilHWIZhGKY8XJT+eAAbjc+bVJtvHyFED4DdAEZZfa5EtZV+3YCqno5hGCbtuCh9v+okIkofIjoZwH4hxFu+JyC6loiWENGS9vZ2B5EcGXF8fMdiGIbpB7go/U0AJhqfJwDYHNSHiOoBDAOww9g+DyFWvhDiTiHELCHErJaWFhe53RjCNXIZhmFMXJT+qwAOJ6IpRNQIqcAXWn0WArharV8G4GkhhAAAIsoAuBxyLKC6nHyXty7slxOGYZjao6jSVz766wA8AWAlgPuEECuI6GYiukh1uwvAKCJqA3A9ADOs80wAm4QQa+IV3YFMPTDhErnetSO8L8MwTA3gVC5RCLEIwCKr7SZj/SCkNe+37zMATildxDI59Bxg00NAtjsxERiGYdJC/52Rq8k0yKXoSVYOhmGYFND/lT6plxm29BmGYWpJ6bOlzzAM0/+V/t535XLbC8nKwTAMkwL6v9J/T9XIXX5zsnIwDMOkgP6v9EedKpcZp0AlhmGYfk3/V/oDVZqgrl3JysEwDJMC+r/Sbxgml927k5WDYRgmBfR/pT/kMLlsHpOsHAzDMCmg/yv98SpTxEf+Llk5GIZhUkD/V/rEA7gMwzCa/q/0OQ0DwzBMjhpQ+srS3/5KsnIwDMOkgP6v9HXJxPX3JisHwzBMCuj/Sp/0v8hFVBiGYfq/0mcYhmFysNJnGIapIVjpMwzD1BCs9BmGYWoIVvoMwzA1BCt9hmGYGqK2lH733qQlYBiGSZQaU/p7kpaAYRgmUZyUPhGdS0SriKiNiG7w2d5ERPeq7YuJqNXYdiwRvUREK4hoORE1xyd+REQ2sVMzDMOkgaJKn4jqANwO4DwA0wHMJ6LpVrdrAOwUQkwDcBuAW9W+9QD+G8CXhBAzAJwNoDs26aMiehM7NcMwTBpwsfRnA2gTQqwRQnQBWABgrtVnLoBfqvUHAMwhIgJwDoA3hRDLAEAIsV2IBDTvkV9TK2zpMwxT27go/fEANhqfN6k23z5CiB4AuwGMAnAEAEFETxDR60TkW8mEiK4loiVEtKS9vT3q/1CckbPkMsvplRmGqW1clD75tNnZy4L61AM4HcBVankJEc0p6CjEnUKIWUKIWS0tLQ4iRUSnV2b3DsNUl+49wDMXAvvWJC0Jo3BR+psATDQ+TwCwOaiP8uMPA7BDtf9JCLFNCLEfwCIAJ5QrdGSoTi67dlT91AxT02x6BNj8KPDmPyYtCaNwUfqvAjiciKYQUSOAeQAWWn0WArharV8G4GkhhADwBIBjiWigehicBeDteESPwLv/LpdPnl71UzNMTVPXKJe9B5KVg8lRtICsEKKHiK6DVOB1AH4mhFhBRDcDWCKEWAjgLgC/JqI2SAt/ntp3JxH9G+SDQwBYJIT4XYX+l2DqBlX9lAzDwCtitPFBoGsn0DgiWXmY4kofAIQQiyBdM2bbTcb6QQCXB+z735Bhm8kx5kzgg8cSFYFh+gTZHiDbCdTHZCjVGdNyHjsRmMu+/aSpjRm5DUOTloBh+gYvXAncNzi/bcMDwPJvlXY8avDWO9aWLhcTG7Wh9Cf5voQwDGOz8TeFbc9fDiz/ZokH5LkxaaM2lH7TaLmc+Jlk5WCYWsNMfTJ4WnJyMDlqQ+kTyQGlwVOTloRh+i7Cnp7j0H+5EaqZcRpCZCpMbSh9QIaMdWwI77NvLbDmV8DdBGx7uTpyMUyayRqpsqJObuzdD7S/4H2m2lE3aaa2foUN94ZvXzgVeFlNN/j9qZWXh2HSRM/+wrYlX/XWsxFzJRY8JPwm7jPVpraUfhDPXgw8ZKcTYpga490fe+sb7pfLtju8NhFR6XOuq1TCSh8ANj0MHLAzSzBMrWH47J+/otCHH1WJF1j6EccEmIrASj8trPoxsG5B0lIwtYwZUw8A2S7rc5nunVorYrTmF3J8sHN70pLkwUo/Lbz2VeDF+UlLwdQy7z+c/7n3IDD2Y97nKO6d3oPAesuIqbUsty9/QS4fHJ2sHBa1p/Sjhp0xTK2w9dn8z6InX1FHsfTf/Efg9b/Jb6s1H/+QI5KWwJfaUfoz/kEuo7xiHtwKdO+tjDwMk3ZEb77Sj2Kp71vtrR/513LZMCQeufoKI4731rPpecupHaW/4ha5tF9hw/jNWOD+hPL2rP1vYONvkzk3wwBSyZvWeRRLveWj3vrIWcDES2vPp2+6wzq3JieHRe0ofc3e95KWwGPbYjnQ4/c28dLngOcuqb5MDKPJWu4dEUHpm4PCVAcgU3s+ffMhabvOEqT2lP6WPyYtgcfzV8ilX5Irhkka273T/ny0fTWZeqX4a8zSN8dAXpiXnBwWtaf0P3giaQk8hh4ply//eaJiMIwvokf+kcqZ8+qXge59jvuaSr9ZpmBIkV+7KogeoHmMXNdjiimgdpT+gHH5yzTw4ZNJS8AwwWhLv67Ja3P1TZuuoLqm2rX069Xg9aDJycpiUDtKf8rn5LJhWPR9KzEA1dvp314sKRzDVAut9DOG0ne9Pk3XRqZJWvq15tMXPUBG1QhO0SB27Sj9CZfK5Z6V0fetxA/Wsc6/3UxFyzBJ0rEBOLAlv+Th/vfd9jWV/tCjpKWfIsVXFbI93ltSiv732lH6XWVMha6EhZJp8G9f84v4z8UwpfDMeUD3rnxLPxvwhmpjhisOGFujln43W/qJMvy40vetxMVKXFCCSRmHfAJoHFHYbvr0XWfl6n7Tb5DLWrP0D24FdrxmZABIz/9eO0q/nNmAFVH6dfEfk2HKIdsN1A8qbDfdO66ZMvUg5szvqoYas/RfvkYuezvkMkXpX5yUPhGdS0SriKiNiG7w2d5ERPeq7YuJqFW1txLRASJaqv5+Gq/4EUib0g+6eQZYef27dlXg3Azjg+gB6gYWtpvunShKP2NP0EqPtVtxhhwmlyffpRrS878XVfpEVAfgdgDnAZgOYD4RTbe6XQNgpxBiGoDbANxqbFsthJip/r4Uk9zVpRKvpUHHHHNW/ucXOPMmUyWy3UC9n9Jv9Na3L3E7Vmc78h4QtRann+0GmkYBw2bIzyl6y3Gx9GcDaBNCrBFCdAFYAGCu1WcugF+q9QcAzCGi/lMb7cFR8V+wgQ8Sy5Lavjje8zJMEEHuHXPwds3P3I6VaQK6dnqfa83S79kH1A8G6gaozweSlcfARemPB7DR+LxJtfn2EUL0ANgNYJTaNoWI3iCiPxHRGWXKWxn0hK2Jlwb36fWpH1oWQa/JVrt54zBMJcl2+7t3RpxQwrE6vRnnQO1F72S75BtSRgVslBM9GDMuSt/PYrc1VlCfDwBMEkIcD+B6AHcTUUHaSiK6loiWENGS9vZ2B5FiJtMEtF4FnPqr4D69B+M9Z5Cln6IBn1jZuzr+75CJF9EDNA4HGkfmt9c1AZftiHasbGf+WECtRe+IXi9YI9MEbHkmUXFMXJT+JgATjc8TANgFZXN9iKgewDAAO4QQnUKI7QAghHgNwGoABZUFhBB3CiFmCSFmtbS0RP8vykX0yhDKuoHA1C8CI08q7NMb9+uZUu4n/rt/e1TuJvmXVh6ZJgvQM+kl2y2zY878Z2tDBpED/Xq78scCas3S370C2POOXB84Pj/sNWFcfslXARxORFOIqBHAPAALrT4LAVyt1i8D8LQQQhBRixoIBhFNBXA4gDXxiF4Gtn9+/wb5VCYCTrnL380Tt5Wij5ex4/Utpe/3ut3X0G8vaUp2xxSSi7ixjAfKyHsj0rE68xVdf7L0dy4DHp4aXl9g99ve+sgTge7dlZfLkaJKX/norwPwBICVAO4TQqwgopuJ6CLV7S4Ao4ioDdKNo8M6zwTwJhEtgxzg/ZIQIuJ7YgXQsbMA8OFTcmkOUJHP1xIll7gL+gaw4/WFABb/L+NzxGLUaaS/3Oz9HdGj0iD7KH1fD24ItnunknH61XYbPjYT6FgLbLg/uM+AQ4HRp8r1xlGpKo7uNC1UCLEIwCKr7SZj/SCAy332exDAg2XKGB8tZwDtz+VffL7FDXwu8Njre2qL3j5XFlj9X8bH/qD0a6w2al9ECKBrhyp+EoPS7+3yMkwCyrgR8jxxBfZle4EFSoVd2ObFxleLplHB27r3yophul/XDmn8+BmUVSZ5CarJJPVcMt07b91c2K+alr4dInfgw3jPkwZqyZfbV1n1Q2kxjzgWhQo+g8hjTQXuHXVPxfnW12Pk9jdr8laLIIPslb+Usmn5mkbJ/zslLp7aUvq5C6+YEvKxRGK3VrVP30q8tu3F8g67b215+1cCVvrpRyvNaV8qtMSzndEtVD0orMntH6erz3gQDTg0xuM60hNQUKbtTrkcfYpc7lkllx/8vvIyOVBbSn/d3XKpFeuBLf79zAv8rEfkMm73jh7cLPd1b+/q/Nz8ugRjmmCln35EVlqkRIXjTDuX5r+RPnaiwwGz+cfR63FeC+ZbQ+zuVwdWfCd8+9QvyOXut+Ry86LgvlWktpR+swoH1Yr26U8EdDQsHZ0Ns1KWPjLerL0gOgPGvjt3yFDIJV/12lLyCpmHeUP213kIfR2zLOIky3BovUouJ35GLne+7nA8y39dCfeOeawkxo38BpBNmfRb/Iyvy+WUz1deJgdqS+lP06l/lFI/YEw3aDLmB5gXqw6p9LMkdGz8hgeiy5KL3qHicwAeDBgw6t4jlx8ar409Hf59k2StMemtXPl6OoD3flJbeVyqgZkgra4JGHG8XG8e41msUTLDFij9Clj6pnunWpa+qdT3vle4fecyufzI//XadJbSlGTWrS2lr38k7QLpMizoAYd462auERdL//mCwCUH9AUbw0/Qsd5bP7BZ5vJOE6t+6K0f+KC8Y739feDVrwDr7y7vOEw+tg9eG0an3e35+KNY6SKL/Gu7wpa+y9tHLOe0Hlo7Xsv/rNO1jP2Y0ag9B+l4y60tpa+/dL/qP83GQNCu5d56kNIv101hWvrnvFzesWxevz7e45XLqNne+q43yztWt0o13bmtvOMw+egYfRtzVm2k/FPVsPQNpV+t60G/Uei3IjtlRW67OUdBPzRZ6Vcf84ewE5nlij0A+PAPxj4B7p0djilmgxCGpT9qNjD9RuCE28o7piYoqiApTGVxMGDwPCopuYH6DXb+e43pkohyXVXbp69dnZVGT5YcopPJWZFOwnooAEY0VDqu2dpS+uYr4APWE3qkkUnQ9PUHWfpPzM7/vPLfIgqjLX01xX3mLeFZPqOQNivY9OOXetMv+Ss5ftITd7ZTBoAayPVRVHlKP8J3byt9/caQ7SpdxsKTeKvVcmnmLHn9BpT1355XDpWVfnLowako6It+65/C+73xt9GOm1N+hqUQV56d9hfiOU5c5Pk9S7zw31WJ6Vb/p1zGqjwYZHusgUaylog4CG/59PVgZjbGlAmmARHXG6TrObUHwDZicpa+j9JPydtpjSn9gLzgs+8M2Un9UCv/JWYr0ydOvz4kdHPbKzGeu4p077PcAmVe+FoxpTE0tS9T4NP3sU6j+PRtS18r/Tjz5OQp/WoFL1gBGLYiFz6WPrt3EmT0bP/2yVZJwlk/Nj4Yls6i4+KTRRjuHU1YvP4Hj7sfe1BrSSJVhAPv538u16c7RGXmfudfyzsOk4+Z/x3wV1RRLP0C944aT+v1CaIoGWNW+8FqpS9R30dGf1e2e0f5/NnSTzl2ruvDvwJ88oXCqJp9bUB7mWkSNH7uHfMmOeya/P7L/zGe81abzephdcZDqqHMC18rnv6QiC5N2ErfT1FFGsjtha97J1ZLX8nWNFrW5K2GUs3NpNfRSEE+fb8UFKz004MdtUAEtJwGjD658JXWjIkHgBOiDuAqdBSAGRJnMvs/gfOWlnbsqBkRK4kOsWw5TS7LtfTN38MMrWXKo8DS98mVE0VhB7l3KuHTrx+i1quhVIso/VCffjpSjLPSL0bTmPzP9o1x1N94n6P8qL1qINJW+mc8CJy7RD54GgoqS1qkw3IIRfs2cz7OmCx9AFj73+Udi/EIUvolW89W7p1KuneCFHAl0OfQ34v9ENvbpmTi6J2+y9DDrc9HepXtB07O3xZl4omOPqmzlP7ES2WlHb/j2UUbUmI5hKL/h6BoB6djmIOJRsqKoLckJjpZ273jk5G25XT341VlIFf719W19fgsT+lWiu0qoGLrM3LZbbm83vqWXObNUWCl37cRWS9rXoeVxrgUpR+quCw3TUEGzXRcRKHo76QcSz/oQZGSXCb9ggJL32egsn5whOPZSl9Z+n6z4UtGW/rq2tq1DHjk8ODucaBDQ2er0OGnzvbvl5fLKyR/VwKw0o+K6Cn88YYdrbaVoPTJZxakZvBU/1mSOVn6gtLPAqDy3AX2xLiBk+QyLMS1L5HtjtntUQK20h+ilKfpYoyUBtyK09fXeZwD8MJS+tVg3xppqLV+tnCbdtlO/fP8a1PfwykpfcpK34WGYd666PUmCmm0y2HXCvdjBrl3TIiAi9aFHKQvKH2tTPRbSynuHUvpjzpJLpNWlHGxcCpwb7NUKElhK/0TfwicudBzNQLR5kYUhGxWUukbcuclOnOkt9Ndrl1vAgPGA/UDgfEXeu3blwDLVQVZOwNsJf73Mqhdpd802r3v5buAY1RZxWxPYUpVXXXowwiVcbJFonc0A8cFbwtye3Ss9ayOxFEDeuVY+vabVf1geby+Piu354BMLbF/k/y8sMo1Xk1spV8/AJhwYX4fnWffrH0beDxb6StrfM/K8uTMP4lcmCHXpbhQ7m0Gfn+aW9/9G7yHn66Bm+0FnjgJePtW+XnSZ/L3YaWfEgZOjNZ/jBrEEj1eSmZd7V4TdMH1HiwcwHLy6Vs0DLcaQhToshvdj1tJRC/yCmvveaeEY1jfa90AGQ3S15X+5keTlsCjIE7fhyO+CoydAwyZ5nDAAPfOyu+XKmEh2ujRg8RA4bXiimsCRaoDxpwp13NuG+uc9j1dCddWGdSu0t/5hpy5etx3i3YFYCRe6/Vew7t25ffRI/cmvZ3AvQPk304jrXDPXrksVjXLZNz5VkOI0u/Y4H7cSpK13DvrSgiztB+mdQPkjVWO0l/8F8Dyb5e+fxzYFvPYjycjB+Cm9ImAhsFuY1dB7p1Y0UrfuIcqPVia7fH+l9wAraXM7Yyfun9Kst/WrtIHgLlrgRk3uPXVN0S2xwtdO1vVvAxT3GbyM1Phrf118X01p6r97MiH380I3qdxWPC2qmK5d0rBtqTqldIvx6e/+i7PB5sUWlk0qcpomabgvpVEZIG977pZyVRXXOn71X/2y9VfLrk5G4bxU07ZRBdL3Cwrqf+nF6/KH0w26wkDnvvpzW+ULluMON2JRHQuEa0iojYiKtCSRNRERPeq7YuJqNXaPomI9hHR/4lH7DIYEOIjD8MsAtH+vFxvHCGXh/1FyI6G332bkdZBuznIYfbsFFWjdOODTqLK46YknFG7d1z+zyBsi75ugLyR+rp7Ryuojz0BDD8OJQ1yx8F7P5VLl+vLSekb9Z81ZhRQXKUudW3ouGrlLvt/xftkDaWv3TbvL8x/wI2/IH8frSeGfqR02WKkqNInojoAtwM4D8B0APOJaLrV7RoAO4UQ0wDcBuBWa/ttAB4rX9wYaD6keB8//OKWte9u0OSC7jnMgcs9b5d27mJMd3xbSQI/t0GUwVwhCpVR957y3DspiZf2BvObZTRIUpPtomSodFH6ufj5gAf9Sls9lIiuwjbrdq9td4QIOhvbQvdDdHsWvvn2oq/F0wPqZY+cFS14pIK4WPqzAbQJIdYIIboALAAw1+ozF8Av1foDAOYQyV+ciC4GsAZAGb9GjMz6kVyGKWo//OqE6tc2XTg6bD8A6Nwe7ZyujDoJmJ/S2bm2bxfwxjNc2P4K8IYqMn3kX8vl8GOU0i/RvfPqV0rbL250veBMg/yOklL6eiBUR6OEEcXSt3/3T6iaFMu+Hk2+IOoHAaNPAwYZQRki62ax++Hy/+f59C1j5sR/L4zc0TSOKKzWlxAuSn88gI3G502qzbePEKIHwG4Ao4hoEIC/B+AzwulBRNcS0RIiWtLe3u4qe2noWYVF89rY+GTK0xd12CBVQcRNJSAfqyolSdf8LH0douiCGRt+yCeB85cDrX+mlE+JSlIXYkma178ml1QHeX0lpPS1BXryXcX7Oln6+h6xrsHhMaYm17KM8kmXvuI77sfIe+t0+P5Nn745EH/op4Ajrwver3mMF9qdMC5K30972O/nQX2+BeA2IUTosLUQ4k4hxCwhxKyWlpawruUzQBVAn3J1tP38LH1NkNL/3dHAq1/Ob9u3FlijXorGfdr9/BMuRqAiL2eQtNKIbKHS3x0hVttMsFbXBAw/Wv4WlIk2AzrNDGpN1tLX36OL+6HUgVwg/uCCrOFqmX0H8Inn5HoUN0reeIDD9997wFP6k6/0PAZbnwvfb/0Cmf5Zv90liMuQ+iYAZlD7BACbA/psIqJ6AMMA7ABwMoDLiOh7AIYDyBLRQSHEj5EUzWOAKw+WkKwrJCd20DRwP//i47O8OP/Nv3M/ffNYoDnggZjmmal2XnUgWg6X9fd462Z0i5PFWYQocsSNnjx3zLdUQQ5K7iGWy4/kMPgfxacfZKTEVRbUrOs77Vq5bBoNTLo8wjGM/6WY0t/4W2mE6BraRMC0L8k5McWqio2dIydvxplwrkRcTMRXARxORFOIqBHAPAALrT4LAWjT+TIATwvJGUKIViFEK4AfALglUYWvqWuKHk0SZulHiZTRCj8qlAkeAD1gP4NThEv8dxhmZtECpV+CZWzedElGOPWqNxjtZqQE3Tt2JtQwolj6fkp/5InA2LOjSBd8DtPSz8kXcp/4Hsf8X4p8/xvuk8sJF3lth10jXUznvRG+r/b1pyAzbFGlr3z01wF4AsBKAPcJIVYQ0c1EpP/7uyB9+G0ArgeQ4nCSUtFpBPyUfjX85yFK4dBzKndaIYCnPg68WWLlLmEU3D75Z7rRfX8zzC2vwlmJ7p0DRlm9JGdI5iJ39KBgRGUVJ7m6rnFZ+gHuHUA+uON4M80NFtuu1agPzyjuHQIGHwZM+ZzX1NwCfGoxMGJm8X1TgtOMCSHEIgCLrLabjPWDAELfqYQQ3yxBvvSgFbvrrLq4b+Awn69vqFlMF1nndmDLH+Xf0d+IPsmmp8OTb5hW4BG+Gz1xCcifbl+qe0f/foMPk3lUksJW+kiBT99Z6ReR068UqKauKZ70yvrttsDSp2jfY557p9gbTHcMM4uTT5KY4hHAtKG+qvYiAzZa2b/3H/GePkzpl+Ij3b8ZeGgcsHNZeD+zMpBLyNmmhcC2xd7nng45dR9A5AfRxt96E+GAfPdOpkT3To/yvTYMk4o3KevaTriXBveOk9J3ecPSlr7P7x1k6b/wWZl8bt09hdv8WDhFLguKtWcQSbGa/8sL88L7ZstR+ukpjs5K3xX9qrrmF+H9tM9411sxCxBm6fso/c4ioa8f/l5GEjxW5LXUnMjkYlk/Oxf4/Sne554OoM56E3G98J+7JP9zXm6R54h5AAAgAElEQVT3Ei19nQZbHysp61orfTLdO0nJot07Mfn0c0rXR70EWfp6wP692wu3+Z5CyWDXrI76PdouvrC6y9luH3eSIymqnsVK3xlHK/W+gXLGaNtPi/fV2fqcTh9iCfpZ+hsf9Mo6+jHAnmoRgHlTlDLF3XTvRKkVuu5ub715LHDRWqBppNGhRJ/+y2oinZ4g5ppdMW5EX3bvFPnec5a8X3hzwKQ6nfX2kIjjU3qypSdgtO9xtTU3oTtk4mC2q4yBWFb6fY8osfD3+8Qjz/CZJXja3YVtYee3L2bKADO+7lkRQ6xScXY5RxPXG8NU9KWkLzCVfpQB791GyopPLQYGt+ZvpzqU5A7R34meK2Df9NUil1rbsPQTd+84XOMuSv+Z8+TSL41ypkmmfXjfSiudG7txUIpmBlmd1yYnX0T3zrJ/kEsd5tk4Mrgvu3dqjHIjdKZ8ThZVNzEHKYue30fpC4G8N5CzHsnfvm9d8PFcreQolr5fNExvh89As8OFr2/my3f7p8yguvISd535kFwOmlT6McrBrvpEGZmqO4m8QDqs1uUad1H6O5fK5UcXFG6ra5JjQ3+6ENj+qiGD+j46twNbny/czyR0cl9ES1+jlX7YNS56Ss8Wyu6dvoj1VUXNdTNoErBnVX6bGY1SFEKhJSjyb9ShRwKjTvY+d4Yk0jIv7jDrI4ql7xfW6efecbF2tO89KPV0KZaxPm9ds0zpQJkEJ8v4TGrvWAe89Dm/zhUWpcfNtQN40Tsuv6FOQW5iDsa/9Pl8GQBZivQPZwC7Q4rt6Pvm43/wkS+im2zAobJOBQUURDERVmGYKOSy9CafI4uVviu2FWR/1snAbI76W+Ci1REVvA8d66UlXaB4LTk+ZaRv7twWfDzhOECbN5BbROm/bRSk+fAP8gLv6fBmvkZ5W+o9KG/goNfpUgZy9Q03/R+kLPVDo9V9jRN7ApOOQlnvYx1XXJYIE+iiKK+gOH2NmbvGvq577agcc5t6UPsZBFHcOz37ZTDDgPGeBR86d0NEu4bz5NJFmJLP8MpK35kiX9XEgOx6J/wLMHhq+afX0Q26xmjYrEfNwZAIHlNhvh2S6lYYN8HG34SKmMfTn/QUWSnunX1rwxVLSdE7VvbHxmFAV0JKX5Ob/6EGEF3S+8ZNthSlH/LdDz4MGBxQUtGcYHfU9XL5/iJZxMXkYIjBomP0m8f4CehuTb/yl3LZsd4zysLe/Gx3ahRyDxVW+n0H8wl/+JcLt7ecDpzmGGNcCqf8XC5zF01ILPTxagAtLOWDefG9GZKK1uy3fXFwPz90cY6CyToOSn99kUHuUkIc7ZS/DcOSs/Tt70ArmyTyAYlet3BNwE3pN44MrqO75mfeekYd608+iQeXhUzq3/O2fAMc1OojXwS3306VOqFzmxdW3BOWQ6cMpZ+z9JOvk8tK3xnjqzL95hoioLXI5I5y0MVfXtPVgkIs/Y/8HxkCt+nh4OO5vmb2GmGfHzwOLGgOD2sz0Zb+xEtVQ4xT0ctx76RB6du/X8tH5TKuZGSRZOnxFHAxXJS+LpHph1mwJWxcIKzKVOc2eT/4DapGSWehEyLufN2b61KpOrbEln7fw/RP2mFiQVzYFqMA6kLWNXdzPs8ARTpgHDD0qJDDOSrMtb8y9snKGOu97xXfL9MEDBgr1xusAuAuN2X9YGDyZ4O3l+Pe0Zd9w5AEi1VbSn/m9+Ty0E8mIEoU946+D0KsaZFF4HX50XutfvB3f4aNgfml6/YEDJdNYz58PvGcFyH2/GUhfv0yfPoZh4HiKsFK3xnjx9ZWmR8fUWWAL90KDDnMv88Fq4BLImbGtF+/9VyAoNjq+sEIvfhdS+T55SYPGyvQZDuBjSosMjfr1NG9c3CbVMZhSj0O906mIdmka4D3ndQPUD7qBG7JkgZyfX6b7n1yJvrON4C6gElMk68ALtCROeo68CthGvY2qesu+8rnaOlvNqq3jjkdaDSKHb0bkEKFffo1hnmRhcXXH/fPUqEH5b4HgKFHeMVcXDGjWO42Lrwgy6OYUlz69/mfzQkvJge3yEG5cUaxZ1eXiJ5glZNdyRrmdgJkiggA2PJ0cB9qiF4j11b61JCg5eWnmBKaoFWK0vebI3H/EGDRMXLdjNIpPIhcvHiVjMknKvTPH9xSoryO32GDMpo+9vvCbYEVruLw6bPS739k6qIrdBdCX2f9moukKbCjKx72mQAlhMwhfuD9fMute1eoqBg4QZ2jVS610tdRF2t/WbBLHvvUrNmPPR7cp2lkCbUJLPdOpj45S99vTCZqhsi4iJJT5rW/ksvF14T3Cw1RNt74Xr5api0ZMRO4aI0sKjP0qPAC52FK3/U71IaL6Vq6Qrn6Nj4YdOLyQzaTfrMEK3134nhCn3wXcOh5pe0bGF0RpPQdslAOPyZ8u86RMmw6MPEyr71YmKOugatzsGil7zIA3L3XiybyDclTNLVIZRElN7ufpb/3vYQKpfsNxEdMIRCbKCXMNH3frqNkEWbpm4pz3xr5OzYMBQZPAY65SUYyde0Ivn5DffqO7p0uZbiY43P1g2Rx9CDZy3Hv6IdgEvMwLFjpuxKHL+6wLwIfW1S8nx9hlo0vIZa+yAL72gpz9djo6JspVwOt84HL98qHT7EUy0d/Qy51MjP9wDIVS1AKBbOGaFhSOD3WEDYBzcZW+nrG8ns/cT9G3Ji/34H3k8kFlO1xzylz3Hf927utAfEwI8n2x3duz1e+485Xxwi6fkN8+qI3P+IsCH0NN1h5soYcEfJAKUPpDz1CLs2Q1YRgpe/KgEOBMWcDc/6YzPkDQ+oCLsKwfPM6d8nAgLwzXbvlhJWcpa5cOw2D5c0ZpPS1hTT+ovz2XKlJ4yYOcs1o5TN5fvirtB4zKZZC2sROLJZUhk0AacjBkkN0u8fpf+T/eutdhpvPHn8ZeWLIQazftWef5xIEZOQZEGyxh7l3di0DtjxV6Ebp3AE8ciTwpMps29shZ/Ta91WmPiSWvgz3jpkWPGFY6buSqQc+8cd46nuWQlSfPjLyBlj29cJN2hIa+3Gg1SfXy2PHAw+3wpvBapw7LLa95TQ5Sa0u4PV4oDFuEBQqqRXzuCJusCal9PetCe+Xd2zLp3/st933jZuweRZt/1VVUZDtcVf6ppI0H+IDrbeyhuEIxud/bjbGwYpFeYW6dxQ6tFnzxMly1q8ugpTt9nfjUH2Rt/oy5ppMv6Fv1MhlUkJU9462ZlfcUrht20tqJZv/mrzjDXlD6fTDtjsEUG8QATeFELJv0KDg6Nlm54BjOOZ2H3EcACq8uUOxHmKjQ0JvK82qHypZjN/vDDWAuH9jdWUREdw7AHCSCmk0rWnbXdcYovT9XDN6TofsoAXz31/0IlB16XDQPVbCtn3GnJkXPy8jv/zCSjMhEV3lpkV2qjpWeVjp9xUaA8JEgy7EMKWpFeXACcD7Rjrmx08A7jEtOR+lT/UBMdp7ga3PAKD8Sl6TAkonhw3SFZMfkK/LQ48orJwUhv3/2NZpNdEP1hEneG06K2XYAHYlyHZHG8jVeXHMQje2SyRsAqOf0h92tNlBHbME946O+Q9Lp7Du18CuN/2PT/X+Sl8I+ea8q0h50TBcgiuqACv9vsLAcf7tW//k3x400NW1G9imlP7wmeFhj9oN1GFYnkEzYV+6Wi73rZYpII77LjB3HXD6ff7H9rvhRBb43fRw+U0GTpbhdUFzDPyObx47NJa8wogsMP7CwmpgeltVZYng3gGAQ1V1qzf+1muzfehhlr6fi2OAMUGrqHunN3iMy2/mq9/10f6C/3hQUBivDjAoy1JX0VkJF1Jhpd/XOSRo2n7AT7vqh15IZbF8K1ufkcuOdV5bUPGS3KAoyZt2xg3+xU9O0TH6Phe+OdnKZbLQtGvl8tlLwvt5J1BL9d2Um+66HPwiUDIJTeDpORBct8CP0T65p2xFaUfFmBT93svw6fvFw+fShjj647v3FLbpN7PZd7odw4+U5NR3UvpEdC4RrSKiNiIqSH9HRE1EdK/avpiIWlX7bCJaqv6WEZHr3cm4Mvxo/3bzptCVjABguU+hkyB0/PpRX8s/bjnWTs6NEGDpm+cpxiSVzro3LDOigXZL7HjVkiUB/BSXtlKrOYEn2ytDRe38SK5suF8dR8msI8JCLX1D6Z/0E+D0+93O9ezFcja66EGg6vJ7cK75hVxe8I6XzhnwH8jXEwjt1M46vn7QFDdZ/dAP+YT9+kWVPhHVAbgdwHkApgOYT0TTrW7XANgphJgG4DYAOkH7WwBmCSFmAjgXwB1EUd4jmaIEpYQwrUidS+SdH3htV0aoGGXGy5er9EP9tWab40voIeeEW5UmeuBuxEy3/pXEbzBSuz2W/j1wICQNQZzsXiEV3Zizou2nffbPXyGX2qd/ys+Bc14Ot+bNAdQRxwOTLsvfTj5urp79XvqO9ue9t1Abve/ybwJ71eDtJpUDavDU/DGUo31Sik+4WC7t/P567CgoZbQLuYd8+i392QDahBBrhBBdABYAmGv1mQtAv7c/AGAOEZEQYr8QuUduM1IVnNxPCIq1N61IPcnq9b+Ry7CwSj9MvzPVydw4dulHZ0Je3aNa+oD8P7KOs3J7lfuo2UqTYcaIVw0fS9+0hx49ojpi6HGbqBbsmb/N/6xnaQ+a5O/+sdFuyc7thdv8krpte7mwXzHaX5TL0afK3D6Zeu+BHxSuq2epm/V7ATnQnGko7+2wr1j6AMYDMGPINqk23z5Kye8GMAoAiOhkIloBYDmALxkPASYO8gYCDUxL31aKgwOyf9qMmg18NuA5/cQpVoPqVyzcMGyQLk/pO1r6mSb3VAxZXWbPsEIHTZbzFapN1senT+QNfHfvqc6An3bLRAnZBPKzr/Yc8GZGN4UkGjTR7p8en9QcLjn7T/xh8DbtLtLnyPbI4AIAGD5DXtN+Vj7gGQCvfy2/PUpSuiD6kE/fb/TDp6qzfx8hxGIhxAwAJwG4kYgK3vuI6FoiWkJES9rbI8ywZIJpf95bt5Wiaa3M/s/gY/i5TbSism9WPQ2/qPIIce88YPiBo1j6rsXNe32UvmuultgJGIycdLmXursavn09eB510pCp3Lc8JZU+1bvPPNW1Evxm7vopfTujalNIWKtOL5KzLx0mc2mCJpVFKR4fRB+y9DcBmGh8ngDATgaf66N89sMA5MUCCiFWAugAUDDyKIS4UwgxSwgxq6XF0VKoRfTr/9EOg7Hm5BT7hjHdCNP+IvgYfuGcQRa4zrx5eJHkZUGWvo4oyvVzvMEyUdw7PkqfHItuxE1Y/pgJKt7BJYdMuZRq6Te3AGPnyPUdbwDbX5E1h13TFEy8WFrcfj7yXPpmwylgX8NhqctzaYyVchVZd7l0P7tyV41Z+q8COJyIphBRI4B5AOwUewsBqEBtXAbgaSGEUPvUAwARTQZwJIB1sUhei4xWLpVD5gDH3QKc81Jw34vM9ASWgnW9eO0cOgC8SybgJipaVSxA6f92otXN0b0Txaev+6XB0g8LO9Thk9VQ+qJEpQ8Ac/4gyztueVpZ+z7++VKwlTZQOB/Fr7iPpqBgiUCk6PQRJxRW84pSRzhYMO9YCVL0m1A++OsAPAFgJYD7hBAriOhmItJa4S4Ao4ioDcD1AHRY5+kAlhHRUgAPAfiKECJCWkQmD7MQw4wbvYeAH4ONgTnbYrcv3qCC7iNPKGzLpSUuU+kXU7QuJRkBGf7n6t7RszTTbunrGc1ptvQ1dU3BkTSl4ufeeeff8vuEKX27YInIuhsRgLomrOszDks/kw5L3+nRJYRYBGCR1XaTsX4QQMF8eyHErwH8ukwZGY3fa68Ldh4S+4JunQe8OL9wP9/c/wHKfvxFMsd665+Fy+JaMnHiZ8K3a+oHSWUurAyIQgBPfUxmRj32m7JNz1cwy/NFKaQdJ2GWvvavp9mnrzEfoPNiktdlIHdAwAx1oNDSD6vZ6y9A4TURh9LvK5Y+kyJ0mKWrpXCpyhd/2LX57e/+u9v+fjN27dKHmoETpPUVFE2Uw1HpFz2OomGwPJZtFXesly6Bt77lta1S8xTM3ECuhbTjJixpWDWrLOlzuFbOstGpLI78WnyT3Wylb9ZzvmI/ML+Ij97ef/tirwSnmwCFTXEoff2APPhheccpE1b6fYkTfwhM/XP39M564lajFYUzeV7pMuhoi4L6tK4FJhzdO67UDZJLPRdBY95YZoEPO7qklALrsRBm6fvkj6mYGOW6d5rL298PW2l/qHL1z/h/soB8sUFZ072jS29Gxro+szFE7+jZ8+//rrzjlAkr/b7EkGlyxqPzxCp1cyz/JrDHmGF4yi9KlyHotdp2rwSK5GDph4WR2mxSk4TsuqadxjiGzqXSNBpovcqSp9wZxiXiF6evqaalnxvILdG9ox+2cSp9XWtBK2ztevQbY/KDDPfOh0/K9TN+437+Svn0h6pEBstvCu9XYVjp92dMJbzy+95F6/fQCK10ZBAYKufqN3VQ+lGqE+n5AnaK5Xd/bBxPXeZ+BcCTUvphlr6e6GT+D5VCz1IuVWnryXh7V8cjDwCM/Zhc2v+/TpFQjIwR/fOGqvTlen0DCHbvlKku6yMktasgrPRrhcYRMtpHx1bbtJzhdpwx6oY89FP57bqASlEc3DtRLCo9Z0HXVdXH3vIH47Ny3/gVzkhK6YcpEa1I1/7Sf3uscpTp3tF88Hj5smiGqBnjHzwml5kGWXXK1RjINAEgoGu7LLsJyPQQUSi4Pl2v7yJMuBgYNqP845QBK/2aQYRPUpn+9/mfz1ns3695tHQFjDi+8Pgulr6Le2eonc8vBL/qR917pFU/QOXYEb3yJu49UJg2N1GlH/Bws11QlaRcn/7JqtB3UN2EUmn9M5kvJ9srZYySBrt+AAABvPVt6X6KXDfBx72Tay8Tqg84dvVgpV9LiBDFnBfRAqu0oe/B8j++v9BncNcPdf62O/KbzWn9rr5b83imPG8qn6lOIiZ6vXz/bVY+dKqT27p9csBUkrDYcW3pVoNyQzYP+4KcWasLq8RFXZOULZcvqUTXyLaXokcV+fr0Y1LUKSiZyEq/ZlAXcuArapRLgeSFq6Ni1v5ahtV1usy7U0p69V35zeZ0/Eg3qY+76N0fyeUENXdw0bGehT/pivzdRa+cSfqso784LooNDNpyVopcyGbKMp7rN7CNaqC+1Hz/3XtKGIANSCUWZawp8NABRYiqCCv9WqF7r4xXDrL0o1zQ2U5g5b8A9w+RE6N2veW+b9B5hJBpbedGqHkLGA8xQ+k3jZJWvrZee/YC794u14/0yZ4IlJa6txzCJmcBwJAjEIs7oRjZbqnw41BocaJr1b6kJvuF1bz1Q0eADT+2tKgbX8s+JqXfB/LpM/2Btp/KZWAYYIkX9O4ViHYRB51HyHDQqANuOUs/m9827jzgkE94TbqQRkGaCLV/HIN0rgiBovlgSNVTrTTZrtJdO5XEtoiHHhltfz0fQ3RHV/p+7p24fgt27zBVJ+iCM5XemXY+vRA2PBDR32mlSgBk7pywt5DQw/n49Hv2AfWD5ZvDvO784wZWGit3in0EcgXaw84Z8yS2IHoPxhtjHxdUL6/VoR+RRX/GXxBxfzNMN6rryuc67Njgk86kBKgu8dw7rPQZhXGhj41QOm/l94Du3RFOY95QSqG9oGYIlxT2Zyn9bI9UZPWD5edMvczPoymYZ5CEpd9b/Jx6265lMpZ+82OVURbv3R7t96sWmTrp3tmzMrg6XBj6+9v2UgmWfqZwNvT2gGi2UuRiS59JBaYCci2EoYk0c9S09JUS264ia6KE5eUOZ9VT1TNETUWvHwC6RF/e/gkofThY+now+7HjgRW3AM+cH/9krWrV4S0FqvfyKW15qoQDGL9n1PGKTKP/NV00g6wDiU0G9GClzyjKGKSKchGbcfK7V8hl7mYq5XK03CB6Cn/DYK+Lzj0UGq+dMkvf5IAqMOOabtqVONwVlcJ8IM78Xgn7G9+tXaCn6L4N+UpfCHntHBZScMj52OzeYVJDGUrfzIJYFOOCf2wmsPrnwO4I0T8FWO4dbenXGZa+Lvvo9yahC7xX1b3jYOlvMYqG6FQJ7/4Y2BzjzFc7SV2aMB9wUz8fff9yfs+MpfS7dsqItbB0zu4HZ0ufSQmUkakYTvuf6PsOU6Xlzn6seF/byln8xejnM7HdO9olYOY5ySl9H0t/0GS5PLglelhgqbhY+jO/K5cNw70JSgDw/GXxydGbYqU/zq+WQxTKVPrCUPoHPpBLsw5DqbClz1SdA+/7txMBn3wWaP1s9GNqt8moYrN4EX80SsYqraeraGUMqz7M0p9oKFHXOgPl4mLpjzwBOOwaOVM62wUMP062x2md62Odvzy+Y8bF1D8HLn4f+EyJJRjjtPQ71ZvsgLGlH1PDA7lM1YnkignATs28UvlcXW60sNm2vSVY2rkqU6r+rZ+lH+bTn/I5bz2O78aF3E1f5PvSE5R6VSz9xEuBYRHyEhVDK/3mMfEdM04GjnMvpmNjPlCnRnybtAdy9YB3cwxKv3O7lxk2IVjp1xoz/qH8Y0y8JGCDw7iAGVUTB1qR9yql/9a35bLOz73jY+mbk8Gq9dqtB7CLpa3INMg+3bvkelMLcLA9Pjn0oHfcv0kaGGC4YmbfGdzPD9vS1wV54lD6O9+QywPJVc9ipV9rTP+78o9RH5AHxSU0LqyY+xF/FV0W7afPdklLfYuqsmQWztZK38+dYr6dVEvp68HjwIenomunlGnbS8C2F4Fdy4HOdqArYlx9bydwcBvQ/oJcano6AFDpyczSjM7UOuZs/7KfYdQNlG5QnVsq9/YYw8PxuH+Sy9X/Vf6xSoSVfn+nYVjxPlEhAs59zWeDw+XUOAI44d/8t534g+iyaPfOhvuAD1U895AjvAFawLtZ9dtAIFVS+nvflQ+gYnnV1xmD6hPmSsUPRM8T9OJVwG9agCdPB569CHjvDpmLqXuPnJNR1TkKVSJTB3x6JXBWhNnlmoHj5fI1bYTocagYvqcRM+XyzW+Uf6wS6Ye/NpOHORA2P8YBpOHHFrY5T4IJKhNYQtionnj1wePA9lfl+nlL81MLaLdOsdTP1bL0258HRp5UPJvoR++Vy/m9wJm/BWbeKj+bGUmLIbL5pSS3vQS8+iU54WvVD9M5Gzcuhh1VWnbOw/6XXOoxntzAewwJ1wZNluMy4y8s/1glwkq/v5OpA0bOkutxWnSZemD0aXaj275dO+OTo2EIMHiqXO9YI99s7LJ04y+Q0S9TvxB+rA33VT7XDSAfPoFlJw0mXyFz1evfTf+fUSJ4dN9jvw3Mut1or1J4al+kaSQw4RJg8+/k9yRitPQB6R4t+tZZOZz+CyI6l4hWEVEbEd3gs72JiO5V2xcTUatq/yQRvUZEy9Xy4/GKzzgx54/AhW3xH9euWepqCZWSbiGM41RM+8F2f//0oMnA+UuLF4bp3FbmRDFHRG9p+ev1zOWOde77aKXfNAo44ivA5VblsGP/KboctcC4c+Vyw/3w0mbElH5aF4hJiKJKn4jqANwO4DwA0wHMJyI7buwaADuFENMA3AZAvYdiG4ALhRDHALgawK/jEpyJQMPgylRjKnhzcLSEDvuiT7nFMtA++4NbCiuAReXJMytf5CLbU1pWTx098uxcmZPHBXuGsn7gdim3n357YPLR7pe9bYalH5PSzzR6IcYJ4HKXzgbQJoRYI4ToArAAwFyrz1wAuorzAwDmEBEJId4QQmxW7SsANBNR1IKVTFqx49pdLaHmMcDx349PDu3j3re6tEgUM+9+9y5g/T3h/Xv2A2/fCuxcGv1cQPGqWUEMPcpbdz23nqymo5wyDfLcOly0P4ZrxoEOgFj5fcRu6R/cIsdWdBRXlXFR+uMBbDQ+b1Jtvn2EED0AdgOwE5d/BsAbQojkHnFMvBxxndUQ5aaI0XduRuroGPgonGyVbnzpc3LwLsi//9ylwNIbpLVdyluB6I1etxWQ+1z4HjBGpb52yW6qUwSbA9uZJi/xXdyutv6CfmPMdgL71sQ7HrZvjVw+ehSw51253rEBWPUj73MFcflPAgpGuvchohmQLp+/9D0B0bVEtISIlrS3xzj5hKksgyYCI4wi5lFuDDNSpnmMz6BwBMpVXH4zUu+pA+7JFGZo3L0S+OAJ73MpU+pFie4dQL7VTFDx/XpyVbFzAfljCJlGb980Vs1KG+v+J97ILjPtxaNHyjfHhycDr/21NCgqjMtdugnAROPzBACbg/oQUT2AYQB2qM8TADwE4PNCiNV+JxBC3CmEmCWEmNXS4hDVwKQI8/kfwdLXVnSmCbh0C3DOC+WJod06x/9r9H3DUi5vs4pn7N+Y/7kkpV/iQK5Gp43u9lH6mx4B7ibggyfl56yPpV/X5O2bxqpZaaH1qsocd/AU4MoD3udNxlyC8RdV5pwGLkr/VQCHE9EUImoEMA+APeNhIeRALQBcBuBpIYQgouEAfgfgRiFEmXc1k04MpR+puLqKXhh+TDxizF0v49iP/Gr0fcPkXmrNYO6wC7eXYAGW6tPX6LkJtqV/4AM5+QoA/niOnMFbzNInVvqBnHCbtx5nziNAvp2e/6Zcf3G+1z7zlnjP40NRpa989NcBeALASgD3CSFWENHNRKQfS3cBGEVEbQCuB6DDOq8DMA3AN4hoqfpLaXYnpiS0xX7u69H20zlv7LDPUmlukSkm4rZc963Jn6TzyrX528Ms/Q0PAguagdU/y28vNXpHE6T0zRz8ALDoWDngDOSPIWSajKgedu8EYs6l+MSz8R9/UGv+57Mejf8cPji9YwohFgFYZLXdZKwfBHC5z37/BIADgfs1SulHjWwYcZxMnTswjsIUFWbD/TIKxm92Z5DSP9ju5b5ffI38u2yHjLUvdSBXoyNubKWvZ9fOXQf84Sz5VvL+I7LNtPTrGr2Mpmzph85bqvIAAAftSURBVDPtWhlt02THpcSAeT2d9B/A+E/Hfw4fyrjyGAblDXD1BYUPAEvsKCUD+//v7ZLFvM3UB5o/XQjMeUZGhMRh6f/pIplu+eSfyZnXXTtke9Noqfjf+DsVcgjLvWOMYfBAbjiz76js8ef1AD174qm/6winYWDK45ibZEzzoClJS1Iepaacti39Rw6XZSB1iufjviNv7IETZZbLBQ3StWJGPUVF55jv2Qes/RWwoF4O3i5T/0OdCjc0ByJt946mYWjpcjDlk6mrqsIHWOkz5TLpcuDyXV6hkr5KOUp/yV8BG38jwzv3b/C2jTlbHjdTB5z0k/z9JpYxljF4KjDtL4ExZxZuG3e+52ozM6za7h1NlRUOkzzs3mEYoPSZqQ8pF5VZanHM2cDRX8+f6XvInPz9yrWwZ/9ULnv2Ax/+QZ7LTkHRONxbN7dtNQYleSC35mBLn2E0l8YwMXDM2cCcp/MVPiBD9C7ZLKOLZv5z+efR1A8EJlzkn3OoYZg3f2GAz/jJp16JTw6mz8CWPsNomkcDl+8F7nfIwX7WozJp3NZnpJtlz7vS7TK4NXifAYcC86qYXZEIuHK/HGw2Z0tf/D6w4jteym2mpmClzzAmDYOBT68Anr8caBwpC574MWCcjD5q/az8PHBC9WSMip0eY+A44KTb/fsy/R527zCMzbDpUvGPneO//fAvAyNjTA3NMFWElT7DBDHyRP/2475TXTkYJkZY6TNMEBMC6phymCPTh2GlzzBhTP1i/udx1ZkqzzCVggdyGSaMk/4DOO4WGaXTsaG0LJ4MkyJY6TNMGHVNwICxwOQrk5aEYWKB3TsMwzA1BCt9hmGYGoKVPsMwTA3BSp9hGKaGYKXPMAxTQ7DSZxiGqSFY6TMMw9QQrPQZhmFqCBJCJC1DHkTUDmB9GYcYDWBbTOJUkr4iJ8CyVgqWNX76ipxA/LJOFkK0FOuUOqVfLkS0RAiR+uoQfUVOgGWtFCxr/PQVOYHkZGX3DsMwTA3BSp9hGKaG6I9K/86kBXCkr8gJsKyVgmWNn74iJ5CQrP3Op88wDMME0x8tfYZhGCaAfqP0iehcIlpFRG1EdEMVz/szItpKRG8ZbSOJ6Ekiek8tR6h2IqIfKRnfJKITjH2uVv3fI6KrjfYTiWi52udHREQlyjmRiP5IRCuJaAUR/XWKZW0moleIaJmS9VuqfQoRLVbnvZeIGlV7k/rcpra3Gse6UbWvIqJPGe2xXi9EVEdEbxDRo2mWlYjWqd9oKREtUW1pvAaGE9EDRPSOumZPTamcR6rvUv/tIaKvpVHWHEKIPv8HoA7AagBTATQCWAZgepXOfSaAEwC8ZbR9D8ANav0GALeq9fMBPAaAAJwCYLFqHwlgjVqOUOsj1LZXAJyq9nkMwHklynkogBPU+hAA7wKYnlJZCcBgtd4AYLGS4T4A81T7TwF8Wa1/BcBP1fo8APeq9enqWmgCMEVdI3WVuF4AXA/gbgCPqs+plBXAOgCjrbY0XgO/BPAXar0RwPA0ymnJXAfgQwCT0yxrxZViNf7UF/KE8flGADdW8fytyFf6qwAcqtYPBbBKrd8BYL7dD8B8AHcY7XeotkMBvGO05/UrU+aHAXwy7bICGAjgdQAnQ05kqbd/cwBPADhVrderfmRfB7pf3NcLgAkAngLwcQCPqnOnVdZ1KFT6qboGAAwFsBZqzDGtcvrIfQ6AF9Iua39x74wHsNH4vEm1JcVYIcQHAKCWY1R7kJxh7Zt82stCuRSOh7SgUymrcpcsBbAVwJOQ1u4uIUSPz/FzMqntuwGMKuF/KJUfAPg7AFn1eVSKZRUAfk9ErxHRtaotbdfAVADtAH6uXGb/RUSDUiinzTwA96j11MraX5S+n48rjWFJQXJGbS9dAKLBAB4E8DUhxJ6wrhFlilVWIUSvEGImpBU9G8BHQo6fmKxEdAGArUKI18zmkOMnfQ18VAhxAoDzAPxvIjozpG9SstZDukx/IoQ4HkAHpIskiKS/U6gxm4sA3F+sa0SZYpe1vyj9TQAmGp8nANickCwAsIWIDgUAtdyq2oPkDGuf4NNeEkTUAKnw/0cI8Zs0y6oRQuwC8Ayk/3M4EdX7HD8nk9o+DMCOEv6HUvgogIuIaB2ABZAunh+kVFYIITar5VYAD0E+UNN2DWwCsEkIsVh9fgDyIZA2OU3OA/C6EGKL+pxeWcv1Y6XhD9IyWAM5AKYHu2ZU8fytyPfpfx/5gzjfU+ufRv4gziuqfSSkD3OE+lsLYKTa9qrqqwdxzi9RRgLwKwA/sNrTKGsLgOFqfQCA5wBcAGlFmYOjX1Hr/xv5g6P3qfUZyB8cXQM52FaR6wXA2fAGclMnK4BBAIYY6y8CODel18BzAI5U699UMqZOTkPeBQC+kOb7KidbuRd6Wv4gR8XfhfT9fr2K570HwAcAuiGfytdA+mifAvCeWuofjwDcrmRcDmCWcZwvAmhTf+bFMwvAW2qfH8Ma3Iog5+mQr4VvAliq/s5PqazHAnhDyfoWgJtU+1TISIY2SKXapNqb1ec2tX2qcayvK3lWwYh6qMT1gnylnzpZlUzL1N8KfayUXgMzASxR18BvIRVh6uRUxxoIYDuAYUZbKmUVQvCMXIZhmFqiv/j0GYZhGAdY6TMMw9QQrPQZhmFqCFb6DMMwNQQrfYZhmBqClT7DMEwNwUqfYRimhmClzzAMU0P8f6naNtKZCF6wAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.close()\n",
    "plt.plot(running_mean(losses_critic, N=1000), color='orange')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 437,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x13791b710>]"
      ]
     },
     "execution_count": 437,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD8CAYAAABzTgP2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJztnXu8FVX5/z8PENeDiIBIgIJCIt5Qjqh5SfMGfP2ClRaWQaaRGn2/Zv5Sv15SKkPxkimmqBlpCqSpqCh5w2sBh0QFBT0ixlFUFJBE7jy/P9Ys57Jn9p699+w9szmf9+t1XmvNmjVrnnPO3vPMWuu5iKqCEEIIsbRIWwBCCCHZgoqBEEKIDyoGQgghPqgYCCGE+KBiIIQQ4oOKgRBCiA8qBkIIIT6oGAghhPigYiCEEOKjVdoClELXrl21T58+aYtBCCE1xfz58z9W1W6F+tWkYujTpw8aGhrSFoMQQmoKEXk3Tj8uJRFCCPFBxUAIIcQHFQMhhBAfVAyEEEJ8UDEQQgjxQcVACCHEBxUDIYQQH1QMzYUpU4BJk9KWghBSA9SkgxspgR/8wJTnnAOIpCoKISTbcMawPbNli1ECP/6x23bddcDmzcDDDwOq6clGCMksVAzbM++/b8rJk922888HWrcGRowAnn46HbkIIZmGimF75rPP8p+fM6fw9ZxVENLsoGLYnnnuufznL74YWLcu/NxrrwEdOwKHH568XISQTEPFsD3Ts2fhPnV14e377WfKl15KTh5CSE1AxbA9s21bvH7BJafPP/cfR80qCCHbJVQM2zPXXRevX8eOwDvvmPqgQcCZZ/rPR80qAOD++43l04ABpclICMkcojW4uVhfX69M1BMD66/Qpg2wcWPh/v/6F3DggeHnoj4nXp+IGvwsEdKcEJH5qlpfqB9nDM2BGTPMA79fP3/7hAn+4yilEJehQ8u7nhCSCRJRDCIyVESWiEijiFwYcr6NiExzzs8RkT5Oex8RWS8iC5yfW5KQhwDYtMmtDxgAzJ8PvPWWv88vfuHWR4/OHWPevOLuOWtWcf0JIZmk7JAYItISwCQAxwFoAjBPRGao6uuebmcAWK2q/URkFICrAHzHOfe2qg4qVw4S4NNP3fquu+aenzTJvwz05z/n9unVy61/8gnQpUty8hFCMksSM4YhABpVdamqbgIwFcDIQJ+RAKY49fsAHCPCgD0Vxe4p3Habv13V/JxzTuEx2rY1G9MAcNFFycpHCMksSSiGngCWe46bnLbQPqq6BcCnAOzrZ18ReVlEnhWRIxKQhwCuCWqbNsVfe/nlwJNPAjvuCKxfb9rCnOXmzs1t8y5hEUJqkiQUQ9ibf9A8JarPCgC7quoBAM4DcI+I7BB6E5GxItIgIg0rV64sS+Bmgd0/OPfc/P3GjMlte+MN4JhjTL1zZ1MuWZLb7+CDc9v+8AfgzTfjy0kIyRxJKIYmAL09x70AvB/VR0RaAegEYJWqblTVTwBAVecDeBvAV8JuoqqTVbVeVeu7deuWgNglsGkT8Kc/1YZZpvVFuOee/P3+9KfcNm8YjBkz3Pqtt0aPM2KEKc89F9hzz1giEkKySRKKYR6A/iLSV0RaAxgFYEagzwwA9tX0ZABPq6qKSDdn8xoisjuA/gCWJiBTZRg/Hjj9dOCBB9KWJD5xlOjdd/uPW7Z060OGuPWzzgq//uWXgbVri5eNkFpB1RhobN6ctiRVoWzF4OwZjAMwC8AbAKar6iIRGS8izmsk7gDQRUQaYZaMrEnrkQBeFZFXYDalz1LVVeXKVDGsd3AwZEQWsWv9rVsX7jt4sFvfeWfgjDPc4xYxPiKDBgF//3tx8hFSS1x3nVl27ds33orBxo3AsGHuHl2NkUgGN1WdCWBmoO0yT30DgFNCrrsfwP1JyFAVtmwxZasaSHxXjGLo39+tf/hhvPGDcZi+9CX/8W23Gb+G+nrgwhzXFkJqi/PPN+V775mXpULKYcAAYNky4IADgMWLKy5e0tDzeXvFKoY4SswuHQUf7pbDDstt+9vf8o85dqyJo0QzV9Icsdb4YUYbNQAVQzFMn27KzZuzv9ZoZzdRD/sgDz5orJHCeOEFt27flN5+O7df1FsUrchILVOKsclPfuLWCyXMyiBUDHHx7iuMHh1viSZNil32GjkS2GOPwv1ucaKW2OWhAw7wn7/++txr6NtAaplFi4q/xru3YJ1Ea4gaWCzPCP/zP2lLUBz//rcpk94POeccYPVq9zgYSiMs45tVUoTUInHD15d7TYbgjCEucd6ms8Rvf2vKSmyUX3yxWw++DdWHRPTduNH4S3TpAmzdmrw8hFSSo44yZdz9ssceq/nkVlQMcWnbNm0JisPGQrKey+Vyww3h7bvtVvja224z/h+rVhmz1oULk5Gp0qxZA1x1VdpSkLSxYWVOO81ti4o8vGkTMHx4zS+fUjHEpXfvwn2yRF1dssrM69tg8X5R8nHNNW59+HBg332TkanSdO5s9lJuvz1tSUiaLFhgyro64OyzTd3r+OnFu+lcw1AxxKXWlkA2b052GalDh9y2qPG3bQPefTf/A7WWHH/CNtRJ88GantbVAf/5j9t+9dW5faM+8yLAihXJy1YhqBjiEqYYsvxw27w5vqlqqXhDZ3gRMTkg+vSJvrZ9+9y2ZcuyE4fKGxX+9dej+5HtH7tft8MOwCGHuO0XXFDaODUAFUNcwhRD0Ps3S2zZUnkP7feDsRIDfPnL8ceaNs2EGzj99PJkqhRMH0JatcpNXxv1IjNtmnlx/MMf3LZSQuCnBBVDXMIUQ5aXl6oxY3jxxfzni4myahMKWTPbNMny/5VUl6Ym//EeewAff+weX3aZW/cqiW9/2+zxeQNPevfaMg4VQ1w2bMhty/IDpBozhkKbsnEC8FmeesqUGzYY66U0qXFTQ5IgYUYn3hS3v/61G23Zu8zkZc2a5OWqMFQMcbj5ZtcawUtz22PwBtu75x7glJy4iMXR2Jjb9o9/pJ9b2quYbMIjQqL45jfNUmNYRkMA6NSpuvIkABVDHKJM0KqhGEqNs/Lee0C7dsnK4s3Mduqp8a7JF1Pq3ntNGTZDSHP/ZrmTqfamm/x+DLUQbp1Uh3xhLvIltKoRqBjiELUkU+kHRffu5gP4yCPFX/vss9lwJGvVCli6FHj+eeCuu8ySkV2XfeQRowDCZghRFk/VwIbw2Htvf/snn1RfFpJN3nsv+tzYsdHnsmJ1VwAqhjh4Y/0sX+7GQam0YvjoI1OWmgSnElPYG28EnnmmuGv69jUxlE47Dfj6192wGXPnmqWarE21ra16XZ0pBw405bRp6chD0if4XS81MJ5VKKrms5/RsNxUDMXSqxew//6mXq09hmKD0P3zn6b89NPkZRk3zo0dUypHH+3Wr70WOOaY8sZLGpupz375v/pVU776ajrykHTZc8/wZdkVK4BLLvG3FbI8uuIKUy5dCkycCHzjG8bg4tJLMxXKn4ohDkH7Y3tcrXgoXlvoOMTNwpYW9k3cUijpT7U591xTWks066h3112piENSwi77DB8efn6XXYBf/crvDf3zn4f33WsvU1pLPruv9sYbRun8+tcmlH9YnpMUoGKIQ1CTW8Xwgx9kw+4+SNeuaUtQGosXR1t2hPHCC5VZs7V7C/36mdJG1fTapAPmc/HvfwNPPlmTJomkANbculBIlLo6s6/gtdoLcvfdptx5Z1OOGRPeb+bM8PYqQ8UQB/sPt7kHbJKeFSuAH/2ocvctNdT3xo2mnDAhOVmSJmxtdc8940eDfewx4IgjivOViMvIkWbT3MaHatHCfPmDywmtW5vosscdV77pLqltbr3Vb7UXxM4YPvoIaGiIzpZo9xVTJpFvlYgMFZElItIoIjmZ30WkjYhMc87PEZE+gfO7ishnInJ+EvIkjl2r//73TenN3lbqxnAc7AO+WOzeh3ctP2t85Svh7fYtHcg/G5g0ya1b89KkeOml3H2dzz5z3xy3bAEefth/nvGUtl/uu6/8MbwvFWHJrCxWgaRM2YpBRFoCmARgGICBAE4VkYGBbmcAWK2q/QBcDyAY5P56AI+VK0vF+OAD/3G10np6FUMxSmLtWlMm7cdQbfItKz36qFvfdddk7zt7dvS5V181joMjRvjbw/Jat2sHDBiQ64uydatxiEoqVwZJHu9Lybe+lezY+b7L3/tesvcqkSRmDEMANKrqUlXdBGAqgJGBPiMBTHHq9wE4RsREJRORkwAsBVBCYtWUqJZi8D5s7MM+Dt/9rimznpfa+/uFbbpZ66Agn39e2VSro0YB3bqFn7MWaUHsPtQrr5ilgs8+M5vXS5YYz1gv1i+G+xLZZfHi5McMCzoZZtCQgVS4SSiGngC8c/kmpy20j6puAfApgC4i0gHABQCuKHQTERkrIg0i0rAy7O2smlTDK/eee/zH06cXP0alYyWVS9euxqJj61Zg991zz4d5V69ebdb+f/97f3tYLKtS2bKl9A38QYOAgw7yGyU88YQp16/PDbEhksxSBUkWuxyb5NJOjx65bQcemNuW9vMNySiGsHjEwcXhqD5XALheVQvGfVDVyapar6r13aLe5qpF8KERtZFUDsEp5Wuvxb/WTn1rIU91XV3uBrINXHbssbn9778/fJw//jE5mdavz81+N39+cWMEvab33tvkoJg4MbcvN66zx+DBpkz6fxN8VnTvntvnvPOSvWcJJKEYmgB4QxD2AhCcM33RR0RaAegEYBWAgwFcLSLLAJwL4P9EZFwCMiVLu3bA+Z598fbtgUMPdY+XLq3cvXs6k6+//jX+Ne3bG2/jWuW550z55JO556KW1KZONfbmSUzD58/PHSefKWIcCm1OV9KIgZRO0ss6Awb4j7t0MZ+NdevcF6GpU5O9ZwkkoRjmAegvIn1FpDWAUQBmBPrMAGANd08G8LQajlDVPqraB8DvAFypqjclIFNyqJo3SBta1/LSS269kklc7Mwham07jPXrayopSA49gyuRHqIcgJ5/3piwJrE2/MEHuTO0oFNeGP/4R/x7DAzYZ1TS7JmUTiVWJxoazJKi3eDeay/zMjdrVvL3KpGyFYOzZzAOwCwAbwCYrqqLRGS8iFjTjTtg9hQaAZwHIMekNbPYTcUddsg9N3q0KSsZ8M1OZYuJT7RhQ21bJOULF+7dkJ45E7j4Yv/5UaMqI1OY8r/ySr+isqEz4hDMZZFFR8nmzJFHmtJ6wSfJ4MH+qL0Wu6S6777J37NYVLXmfgYPHqxV4/XXVY1uzz03Z45pf+SR5O87ZowZ+4MPou8fxbHHqh56aPIyVZNddlHt2jW3/bTTzN9i+nS3zf59ANXWrVWHDi3v3oDqKafktt93n3sf72ewf3+/DHF+1qxRXbnS30bS5803VXv2TO9/UuH7AmjQGM9Yej4Xwk75bwpZ4apkzKSOHY2deylLQuvX1/aMAQC+9jVjfeQNVLhtmxtaICqQ36ZNwOOPA0OGlHZfa3EWFj3zG99w6zaXBGCskMLIN5Ps1MkYMdiAhyQb3HRT/pDazQQqhriE5S+25qCViIqoapYvvNYxcc1kN2zItaqpNTp0AN591w1LAfiDlbVv79bXrgV++Uv/9fPmAQ8+WPx9reNcmJVTixbu+713M/qGG3L7vvpq7sbljTe611sOPrh4GZs727YZc+5K2PsnafZcDpWIjFwEVAxxCfNStW+Elcj9vGmTcVDzPuCDHrSPPhoec6jW9xgA1/lL1c3BPG+ee977+3XsCFx+ee4Y3jf8uNj/5U47xb8mzOfBWp+88gowfjwwZUp0JkBLMU6MzZm77zZGGZWIBZaVLH077pjq7akY4hL2oLAzhkq9uQQf7n/5i//4xBNzzd8AYNEi/9t1LeINxV1XZ2ZPxx3ntsUNnvfqq8ZiKS42+uWf/hT/miCHHupuoO+3n4m1P3p0Yeu1rCUsyiorVpjy0kuTHzvtxDnWAbISwSGLgIohLt4lDUslZwxhy0HeYHHeNJNhsVdoF2/Yf39jYfLKK/H6279lqRm6gNwQGMVgQ3yTaKxiAPxv+Nu2GZNhVRPwsLGx+LFLuSZJbNrbNHOeg4qhPCqpGD79NPfh5LV99wb2GhfiE1jKMkqW+PGPi7/mf//XlCefnBt6IngchVUMpfqB9O8fHsojLlkOlZ4FVP17Oh06uMuOEyYYk+EHHzTew6U4Ja5e7dbTiE6cL77ZxIlmFaEKy11UDHHo2dNdYvBSScXw97+7QbfmzDHlWWe5bxLe5ZGgTXyrVuFLTLVEvqx1USEDfvc7M9P6619zFYvdpyhEqYrhiSfM2+qbb+Z30Avjv/+7uP7F8MILZglre8k+F7b23rmz8Ra2Pi3lzNi8HHFEMuMUQz4fnsWLzee7Cs6rGY+yljL2IXzmmeHnK6kYAKCpyZS77WbKdeuMsli4MHqqqWr2PLIeWbUQImZfx6ZA9JJvE9d+aYIbwi++GO++9qFS7L5RWFynuDz4oN+0dePG/F/+Tz4xoRTiYB9uo0ebkCFxr8sqURv05czSLF5rsSefNCbTWcJaylXSodaBM4Z82LfHKNPPSisGi9c0UwQYNiy3j32QWdPZfG8etULUVD5OCJIwT/U4ZsX2wVnNGVeLFv7ggG3bhltZAcBDDxml16NHrpVakGDAtscfL0vM7R4bBRcAjjkmvejEo0b5E1alABVDPqxNc5Tpp1UMP/1psvcNKhrvBzQquqhdKrEy29lGLWP3DLyIlB4SO04U3K9+1fy/wxRLJfnmN/2zjisiItGfdJIpP/ig8AZ5MB7TaaeVLl8WKGVDthjrvPMzkkCyY8fUrQqpGPJhvW6jZgyVeqMIbi557xP2sATcKbZ16rrlluTlqjZha7ybNsW3GPr4Y//SQ5wp+Natfm/rahKMl/TUU+nIkVW8+ZBfeCGeaem0afHHHzTIlGln1mtsBD780D/DzZfmtgJQMeTDvn1HKYZKbQLZfMg2rEOcB5oN/W2vLSZMd5ZZuNB/XIwy7tLFKBGrTONEXn3kkfjjJ81DD/mPy9m38OKd8cbdhM8iXgfHww4zn/VCPirFKFe7/Bp3P6pS2ICZ3uRVVc72R8WQj7QUg80xbZcC4ji72LVxK3Opyy1ZwxuKpNQwASef7C+zyoABwPHHF3dN1P6W9w3Tu7dQTO7wrGE/017lffjh/tAnc+f6v69xcxuoupu7WfnuePcJi0nUlQBUDPkopBi8JBXbxPuFLiaYl512FiNzLWBnS61bl77uH/d/U+XpeiizZkVnowuzXw+z2gL8ysCGkAYKb1inTb7/gf39g0uJI0caBbltmwlouH598Y5q3r952hkif/MbU3pzdFTZQoqKIR92rTlO3KFScjLnuyfgD5FQyMpoe1UMIibi5csvlz5GVPTTIBlIwg4AOP308PYw7/vHH3f9XLwMHx4+RjE5I6pN9+5mdhz1f7DLYGF/hxYt/NZq3rS2cfaMokzS08DOGt8PJsIE8KtfVUUEKoZ8FKMYkkrv6fW8/M533HpYaO+ePd0v+syZptzeFANg/BaCFjbF4HVOzPfwv/PO0u9RDeyD75pr3LbRo4FDDgEOOCD8GvvCYsOUZzmktN1cjtpUtkusxZpiJ+XwVi1szvOwUP+XXFIVEagY8mGnl3Ee+mH/xFLwKoYTTvCf+/73/cfvveduUFrzRrvBWsupPSvBiSeaMt+UvJQwHJXGa7Zol1l+9rPcfgsW+I9tKlibAbBHj+RlqxRRxhbW8KDYJcU4/hvB71qaZGCPg4ohHza0gfU8DmOEk700qbVbr5lpcMo8cWJu/7PPNqUNE2HzCVTbDj/r2P/TSy/lWv8EsSlbs4D9Pz77rNsWZYzgXZ8PBg20kXmLyR2eFtYSbfZs//7Q7NmmTDqk/GefufmWsxD6vAqezYWgYsjHPvuYctddo/vceKMpy4mKuWSJWSa45RZg0iS3PfgACJtC22USEbPP0NBgjovJJ9AceOklt26dxILYJYff/77y8hQi6N0dzFgXZsqaz3pNxLwV18JM8pRTjEI4+mh/zmUb7ymuYrBLT4XwbmaXE1W3UqQQaZWKIR92Qzdf3CH7sM6nPAphwy/Yt/9C9/LSooX5olx7rV/OOGEjmhN77+3Wo5YNrN14FvIiFMokNmNGePu2bdG+GDvsEG3FlDbe2U6bNq5Zrc2L4X2Tj6sYundPRLTUKZTgqQIkohhEZKiILBGRRhG5MOR8GxGZ5pyfIyJ9nPYhIrLA+XlFRLIVK9pu+OZTDNWKl+S9l8Was3ljKZFwvBFZozY3vZu6aeP9zK1b5+6N2LfHdu3Mw/Q///Fbr7RsGR2tde5cY8aZFesrL16HzJ//PNffohgLvTBsnoMwsjhL8PLqq1W/ZdmKQURaApgEYBiAgQBOFZGgCckZAFaraj8A1wO4ymlfCKBeVQcBGArgVhHJTsRXqxjyfRDthljSiiFsz8L7prR+PXDrrdF9iZ8WLYBly0x92TLzgN24MTupHMOwn7u6OrPftcceuTPBurrojeWxY/3H775rymAmwLSZO9dvgXflleXNwMPIZ+ZplVAWowV8/LG7DPrcc1W7bRIzhiEAGlV1qapuAjAVwMhAn5EApjj1+wAcIyKiqp+rqn19aQsgAx5GHtKaMXTvHm6r7X0otG3rHteyN2s18RoRtGxp/ob275xyxqxQvLFy3nwTePvt6L52r8tLVE6LH/ygLLES59vfjt+32LfnO+4o3Md+z7PoGf/AA269ioYDSSiGngA8OSfR5LSF9nEUwacAugCAiBwsIosAvAbgLI+i8CEiY0WkQUQaVq5cmYDYMXjnHVNWWzHE+TCT5OjUCRg/Pm0pcvG+8VujgijCsvgFN6Pnzi1fpkpQyBLIzh4GDAD23be4sX/4w9JkygreWFBVtDRMQjGE7XIG3/wj+6jqHFXdG8BBAC4SkVDPLFWdrKr1qlrfrVou69Z0NF/gtkoohnyBzubMMZEXSXKsXRsd5jpN8q2Lh2GXyqKwARazhtd3Jwyb67xcP5O4eb+zREqZ95JQDE0AenuOewEI+nJ/0cfZQ+gEwGceoapvAFgHYJ8EZEqWfBY+lVAMdXXR54YMyU0zStPU7ZNiU4TutpsbdPDee3PPezdwsxAXKsiNN+bui3gpNGuK4rDDTBnm9Z1y3oNIqhw0L0gSimEegP4i0ldEWgMYBSBoSzcDwBinfjKAp1VVnWtaAYCI7AZgTwDLEpCpepSrGILX/fa34Rna8hFcHy/2TbM5cfDB+c+/+WZ15KgUixebvA6jRuXvF2awcMkl5iUomBeiWnTtmn92Xuqmuf0+hH1HH3ustDErTcoOqmUrBmdPYByAWQDeADBdVReJyHgRcdxNcQeALiLSCOA8ANak9XAAr4jIAgAPADhHVT8uV6aqYtdx46SNDCO4cXz22cX7IATf/rJufpcm3mBpixfnbtD2719deQphLYmKoXfv6HNf/rIpvWksLTaqZ1pmu127GkfRqBmwffMvFvviFDaLstZQSeSMTpLgqkCpv3uJJOLHoKozVfUrqrqHqv7GabtMVWc49Q2qeoqq9lPVIaq61Gm/S1X3VtVBqnqgqj6Y7z6ZRMRYt5RqGWRd/idMMFYnpThXeTNbAcmHDNiesB7EffqYZZezzkpTmsIkbbZpnSjzBVkMs3CqFN4XqqOOAnr1ip6xlKqw9trLlPkUZjmRCypB0EPdOvpVCXo+R1GME1C7dqWng7zQmTzNmePP2FQMrVvnJish4fTrB1xwQfpZuoohSeVgPar/67+SG7McvCEv7DJShw5mFhz0MRk8uLR7WGOVfHtxXs/4LBBcNejXr6q3p2KIopjEO23blq4Y/vxnU5a6sWbxzhKGDi1vrO2dCRPcJZVa4IYbTJmEwv/ud8sfI0nyWQq1a+c3ZS01x3q7dmYvMN9Gc5wsic0I/jWiKGadf8UK4Pbby7vfzTeXd713AzqrZolZxYZlzqr/yEknmY3TuAmH8hH15hmW76MaFJq5dewI/N//mXqp8b9EzN/vjTdKuz4tJk82ZVTQxwpCxRCFtWAoZtPnpz8t/X7lpu7zfmlKiSXTnDnhBLN0kWVnqKTeaMOcNTdtyl3Tnj8fuPvuZO4Zh6iggIDZFE/CvPZvf/MfZ33J9Uc/Mr+31/u5SlAxRGH3GILJcfJRTrKefL4Lcfj61936LruUNxZpPtx2W3go7vr6eJ/9ckKJeB/23rzU1aKam+w1BhVDFFYxVCtpRrlhsr2zhKCpGyFhbN0aHU/JsmJF9LmbbzbfD5HS3ui9Hv5phDq3+3L/+lf1751xqBiisGZ0lV6WGTDATb9YLgsWuBncCInit781ZatWwMsv+88FY0addlr4GOPG+fMElLLUlUb+aa9j3223mXLHHasvR8ahYoiiGophwwbjZJVUuN/99weGD09mLLL9ks8B0gaOtDz9dG6fRx/1ZxosFbsMlcRYhbDB96yFl5e+fSt//xqDiiEKG9WwkrkOCgUPI6QShIVbeOIJsxwUFgL7ySdds9FNm4ATT0xGDutTUWxMqFKwe3CXXGLKTz6p/D1rGCqGKC6/3JT//GfhvqUu34x00lZU442JEEtYxj+bQzosD8hxx5k9gNtvzx/5t1isYsjnhZ0Ul17qP67FSKtVhIohimLWP73LN3feGf+6efNMWa5FEiHFEAwm511eOfxw4wDY1JR73Y9+VHwkXxEzZhjVVAxduviP7Xdu2rTK37sGoWIoRLFWSdddV/w9guu6hFSSoGLwmlmLmJAhpSzviPjfxO1SaZQTm40WUA3FEMSG26AFXyhUDIX4+c+L619KEqGkA6URko8TT/QbVUSFIn/22eLHHjTIrXszLb4fTNECd8ZQraCPRx/tOvjZJbGwZTVCxRDKx57I38W+UZRiekqrCFJNOnb0B6gLSwsKlO90tmaNWw+bgdjAj6XGQCqWTp3c0B/29w/bUyFUDKE89JBbj/uhtUHw4vb3eoyGeZ4SUkm8n9P6+uh+qsAhh+S2H3EEMGJErtVenz5uPRgd1cv69W7aylJD1heLXbrasIEzhgJQMYThfdOJ+6C3X4i4UVbfesutUzGQNNi2zfwU2kcLS150443mBapDB7+vg12iWruJ8p+lAAATF0lEQVTWzf1gsSG21671P5C9y0+VxC6NDRsGnH66qTN3SShUDGE89ZRbj7v5bD/o+d6SvMyf79bLiTdDSKmIxAvFcuWVJpT7lClu2/77u/Wjj3ZDYtgXnk6djPOmlxtuMMYZwSWqcsPBxMVmqJs9223jjCGUKi3u1RjePLBh0SjDaNvWfMDj2nl/73tunbHgSZbp1ct8J1SNGWs5UWiLNeZIkh/+MPf+3GMIhYqhEHHfZkRKz+R24IHFX0NItRFxcyPk44MPKi9LKYSFAqlWkMwag6+qSdK+ffylJEsxDnGEZBn7kP3lL/3tL71UfVnCCCqBffZJR44aIBHFICJDRWSJiDSKyIUh59uIyDTn/BwR6eO0Hyci80XkNaf8evDaVNhtN1MWmyAjzoxh2zb/LISbX2R7wUZm/eMf3TZV4NBD05EnjDPOcOv33pueHBmnbMUgIi0BTAIwDMBAAKeKyMBAtzMArFbVfgCuB3CV0/4xgP9W1X0BjAFwV7nyJIJd/y82pV6cGYON2mqppdzDhOTDejDbXCaLFuXvP2wY8K1vVVamILffbhzvZs3ijCEPSewxDAHQqKpLAUBEpgIYCeB1T5+RAC536vcBuElERFW9weAXAWgrIm1UtUqGzRFs3lzam3z79oVnDEHFcMQRxd+HkCyyapX/2Gvxc+KJwCOPuMfvvw/06FEduYJ07Qocf3w6964RklhK6glguee4yWkL7aOqWwB8CiAQ1QrfAvBylFIQkbEi0iAiDSu9rvaVYNOm+NZIXtq1K2yVZN+mCNnesOagFq+z28MP+zel01IKJBZJzBjCzHaCef7y9hGRvWGWlyLVuKpOBjAZAOrr6xPIDJ6HUhVDnE224IyBkO2FVq3yv/h0715aClBSdZKYMTQB6O057gUgGDHriz4i0gpAJwCrnONeAB4AMFpV305AnvLZtKm8zG0ffhh9joqBbK+8+64pJ0xwYxKRmiQJxTAPQH8R6SsirQGMAjAj0GcGzOYyAJwM4GlVVRHZEcCjAC5S1YjYvClQ6ozBMnVq9DnvG1VYtixCapUvf9nMCC64oPK50klFKVsxOHsG4wDMAvAGgOmqukhExovICKfbHQC6iEgjgPMAWJPWcQD6AbhURBY4P+kHSN+8ubwPdj5rh48+cus2XgshhGSIRDyfVXUmgJmBtss89Q0AcuJRq+qvAfw6CRkSpVTFcMUVxrln+fLoPmeeaco2bUz8GUIIyRj0fA6jVMVgo0TmS3BiTfpGjSp+fEIIqQJUDGHMmFFasvABA0yZz6PSej3TOoMQklGoGJLE5m7Il3hkhx1M2bVr5eUhhJASoGJIEq8lU5hZalMTsHChqf/sZ9WRiRBCioSKIYx27YD/9/+Kv84bAiDMA/qgg9w6ZwyEkIxCxRBk+XIT7+j++4u/1qsYdtkl97w3JIANOEYIIRmDiiHIggWmXLq0+Gu9D/tqJTgnhJCEoWIIsnZtedeXMtMghJAMQcUQZKedTHnNNaVdny+HrHVoO++80sYmhJAqQMUQxMYyOuqo0q5vbIw+t26dGffaa0sbmxBCqkAiITG2K2yEyFI3h/Ml3nn++dLGJISQKsIZQ5Cf/tSUbdqUdr33Oq93Mz2dCSE1AhVDFKUqhn793Lo3Jn2hzG6EEJIRqBiiaFHin6ZlS+Dqq03dqwzs3sV3vlOeXIQQUmGoGKKw1kmlMH68Kb0JezZsMOXXvlb6uIQQUgWoGIKMHAnst58Ji1EqVjHsuafbZh3eSl2iIoSQKkHFEOTzz/P7IsRh//1N2cpj9GUVA0NhEEIyDhVDkHXrylcMNsmPN8IqZwyEkBqBiiHI55/7g+GVQphi+PhjU1IxEEIyDhVDkCRmDO+9Z8oJE9y2b3/blN4Iq4QQkkESUQwiMlRElohIo4hcGHK+jYhMc87PEZE+TnsXEXlGRD4TkZuSkKVskpgx2NnBc8/ltpWSS5oQQqpI2YpBRFoCmARgGICBAE4VkYGBbmcAWK2q/QBcD+Aqp30DgEsBnF+uHImRxIxhv/2iz9EDmhCScZKYMQwB0KiqS1V1E4CpAEYG+owEMMWp3wfgGBERVV2nqi/AKIj0ef55YM2a8mcMhx5qygtzJk9AfX15YxNCSIVJQjH0BLDcc9zktIX2UdUtAD4F0KWYm4jIWBFpEJGGlStXliFuHo480pTlKgYA6NwZ+Oyz3PZ99il/bEIIqSBJKAYJaQuul8TpkxdVnayq9apa361bt2IuLZ6//KX8Merq/Iqhc+fyxySEkCqQRNjtJgC9Pce9ALwf0adJRFoB6ARgVQL3rgxLlpQ/RseOwH/+4x4fdphrrUQIIRkmiRnDPAD9RaSviLQGMArAjECfGQDGOPWTATyt2gx2YV9+2a0nYe1ECCFVoOwZg6puEZFxAGYBaAngj6q6SETGA2hQ1RkA7gBwl4g0wswURtnrRWQZgB0AtBaRkwAcr6qvlytXSey2m0nUs359+WO97vwKS5cCu+9uFENdXfnjEkJIhUkkg5uqzgQwM9B2mae+AcApEdf2SUKGROjdG9hjj2TjGa1ebcrFi4EhQ5IblxBCKgRTe3p54YXkx7SB9NasAV58MfnxCSEkYRgSo9Js3erWmcWNEFIDUDFUmi1bgMcfT1sKQgiJDRWDZfRoU/7iF8mOu2EDMGJEsmMSQkgFoWKw3HWXKb2+B+Xw/e+bcuJEf/htQgjJOFQMQebMSWac00835WuvuW0TJyYzNiGEVBAqhiDLlxfuEweb3vPdd922ceOSGZsQQioIFUOQo49OZpyOHXPbmO+ZEFIDUDEESSqRTnCcPfdMZlxCCKkwVAyAf3N4/PjK3OO00yozLiGEJAwVA+DGRrrmGhPXqBJIWORxQgjJHlQMgKsY2rVLdtxnnnHrTU3Jjk0IIRWCigEA7rjDlBsSzjA6eLBb/8pXkh2bEEIqBIPoAcDFF5ty8eJkx+3YEXj7beCee2iqSgipGagYvJxxRvJj7r47cMklyY9LCCEVgktJK1e6dYauIIQQKgZcdplb79MnNTEIISQrUDFs2+bWe/VKTw5CCMkIVAzDh5ty9uxUxSCEkKyQiGIQkaEiskREGkXkwpDzbURkmnN+joj08Zy7yGlfIiInJCFPUbRw/gR1dVW/NSGEZJGyFYOItAQwCcAwAAMBnCoiAwPdzgCwWlX7AbgewFXOtQMBjAKwN4ChAG52xiOEEJISScwYhgBoVNWlqroJwFQAIwN9RgKY4tTvA3CMiIjTPlVVN6rqOwAanfGqh2pVb0cIIVknCcXQE4A3iUGT0xbaR1W3APgUQJeY11YHxjIihBAAySiGsCdq8DU8qk+ca80AImNFpEFEGlZ6fQ/KJekcz4QQUuMkoRiaAPT2HPcC8H5UHxFpBaATgFUxrwUAqOpkVa1X1fpu3bolILbDkiXJjUUIIdsBSSiGeQD6i0hfEWkNs5k8I9BnBoAxTv1kAE+rqjrtoxyrpb4A+gOYm4BM8fBGP+VeAyGEAEhAMTh7BuMAzALwBoDpqrpIRMaLyAin2x0AuohII4DzAFzoXLsIwHQArwN4HMBPVHVruTLF5tZb3fqjj1bttoQQkmUSCaKnqjMBzAy0XeapbwBwSsS1vwHwmyTkKJqDDgKmTTP1oUNTEYEQQrJG8/Z87tjRlLNnA0OqayVLCCFZpXkrBkv//mlLQAghmaF5KwZuOBNCSA7NWzFY6NxGCCFfQMVACCHER/NWDBs2pC0BIYRkjuatGO6805RbtqQrByGEZIhE/BhqjqYmk4fB5mDYeed05SGEkAzRPGcMvXsDPXsC9fVAhw5AmzZpS0QIIZmheSoGyw030GSVEEICND/FEAyz/fnn6chBCCEZpfkphokT05aAEEIyTfNTDIQQQvJCxUAIIcQHFQMhhBAfzdOPAQCOPRbo3BnYZ5+0JSGEkEzRfBXDk0/SVJUQQkLgUhIhhBAfzUsxfPihWz/99PTkIISQDNO8FMOqVW69RfP61QkhJC5lPR1FZCcReUJE3nLKzhH9xjh93hKRMZ7234jIchH5rBw5YuNVBp98UpVbEkJIrVHua/OFAJ5S1f4AnnKOfYjITgB+CeBgAEMA/NKjQB522qrDxo1u/cEHq3ZbQgipJcpVDCMBTHHqUwCcFNLnBABPqOoqVV0N4AkAQwFAVf+pqivKlCE+jItECCEFKVcxdLcPdqcMS2zQE8Byz3GT01Z9TjghldsSQkgtUVAxiMiTIrIw5GdkzHtISFvRDgQiMlZEGkSkYeXKlcVebjjzTFNeey2wbVtpYxBCyHZOQQc3VT026pyIfCgiPVR1hYj0APBRSLcmAEd5jnsBmF2knFDVyQAmA0B9fX1pnmnXXWfKESMACdNXhBBCyl1KmgHAWhmNAfBQSJ9ZAI4Xkc7OpvPxTlt6tGyZ6u0JISTLlKsYJgA4TkTeAnCccwwRqReR2wFAVVcB+BWAec7PeKcNInK1iDQBaC8iTSJyeZnyxIOKgRBCIhGtwXhB9fX12tDQUPyFdvmoqcnkfCaEkGaEiMxX1fpC/Zqn+y9nDIQQEknzVAzceCaEkEiap2Lo3j1tCQghJLM0T8VACCEkEioGQgghPqgYCCGE+GheqT2feQZYvrxwP0IIacY0L8Vw1FFpS0AIIZmHS0mEEEJ8UDEQQgjxQcVACCHEBxUDIYQQH1QMhBBCfFAxEEII8UHFQAghxAcVAyGEEB81mahHRFYCeLfEy7sC+DhBcSoJZU2eWpEToKyVojnLupuqdivUqSYVQzmISEOcDEZZgLImT63ICVDWSkFZC8OlJEIIIT6oGAghhPhojophctoCFAFlTZ5akROgrJWCshag2e0xEEIIyU9znDEQQgjJQ7NRDCIyVESWiEijiFxYxfv+UUQ+EpGFnradROQJEXnLKTs77SIiv3dkfFVEDvRcM8bp/5aIjPG0DxaR15xrfi8iUoasvUXkGRF5Q0QWicj/ZlFeEWkrInNF5BVHziuc9r4iMse55zQRae20t3GOG53zfTxjXeS0LxGREzztiX5eRKSliLwsIo9kWVYRWeb8fxaISIPTlqn/v2esHUXkPhFZ7HxmD82irCKyp/P3tD9rReTcLMr6Baq63f8AaAngbQC7A2gN4BUAA6t07yMBHAhgoaftagAXOvULAVzl1IcDeAyAADgEwBynfScAS52ys1Pv7JybC+BQ55rHAAwrQ9YeAA506h0BvAlgYNbkda6tc+pfAjDHuf90AKOc9lsAnO3UzwFwi1MfBWCaUx/ofBbaAOjrfEZaVuLzAuA8APcAeMQ5zqSsAJYB6Bpoy9T/3yPXFABnOvXWAHbMqqwemVsC+ADAblmWteIPxiz8OH+wWZ7jiwBcVMX794FfMSwB0MOp9wCwxKnfCuDUYD8ApwK41dN+q9PWA8BiT7uvXwJyPwTguCzLC6A9gH8BOBjGEahV8H8OYBaAQ516K6efBD8Htl/SnxcAvQA8BeDrAB5x7p1VWZchVzFk7v8PYAcA78DZJ82yrAH5jgfwYtZlbS5LST0BeJM9NzltadFdVVcAgFPu7LRHyZmvvSmkvWycJYwDYN7GMyevszSzAMBHAJ6AeWteo6pbQsb+Qh7n/KcAupQgf6n8DsAvAGxzjrtkWFYF8HcRmS8iY522zP3/YWZIKwHc6SzR3S4iHTIqq5dRAO516pmVtbkohrD1tiyaY0XJWWx7eUKI1AG4H8C5qro2X9ci5UpMXlXdqqqDYN7GhwDYK8/YqckpIicC+EhV53ub84yf9mfgMFU9EMAwAD8RkSPz9E1T1lYwS7R/UNUDAKyDWY6JIu2/K5x9pBEA/lqoa5EyJS5rc1EMTQB6e457AXg/JVkA4EMR6QEATvmR0x4lZ772XiHtJSMiX4JRCn9R1b9lXV5VXQNgNsxa7I4i0ipk7C/kcc53ArCqBPlL4TAAI0RkGYCpMMtJv8uorFDV953yIwAPwCjdLP7/mwA0qeoc5/g+GEWRRVktwwD8S1U/dI6zK2u5a2a18APzdrEUZtPObtDtXcX794F/j2Ei/JtOVzv1/4J/02mu074TzHpqZ+fnHQA7OefmOX3tptPwMuQUAH8G8LtAe6bkBdANwI5OvR2A5wGcCPMm5t3QPcep/wT+Dd3pTn1v+Dd0l8JsDlbk8wLgKLibz5mTFUAHAB099ZcADM3a/98j7/MA9nTqlztyZlJWZ7ypAE7P6vfKJ2u5H/Za+YHZ6X8TZi364ire914AKwBshtHsZ8CsGT8F4C2ntP9cATDJkfE1APWecX4IoNH58X646gEsdK65CYHNuCJlPRxmCvoqgAXOz/CsyQtgPwAvO3IuBHCZ0747jHVGI8yDt43T3tY5bnTO7+4Z62JHliXwWHJU4vMCv2LInKyOTK84P4vsWFn7/3vGGgSgwfkcPAjzsMyqrO0BfAKgk6ctk7KqKj2fCSGE+GkuewyEEEJiQsVACCHEBxUDIYQQH1QMhBBCfFAxEEII8UHFQAghxAcVAyGEEB9UDIQQQnz8f5WFmAJ1EvgcAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.close()\n",
    "plt.plot(running_mean(losses_actor, N=1000), color='red')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Actor-Critic with Experience Replay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "l1 = 4\n",
    "l2 = 150\n",
    "l3 = 2\n",
    "\n",
    "actor_model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(l1, l2),\n",
    "    torch.nn.LeakyReLU(),\n",
    "    torch.nn.Linear(l2, l3),\n",
    "    torch.nn.Softmax()\n",
    ")\n",
    "\n",
    "critic_model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(l1, l2),\n",
    "    torch.nn.LeakyReLU(),\n",
    "    torch.nn.Linear(l2, 1),\n",
    "    torch.nn.LeakyReLU(),\n",
    ")\n",
    "\n",
    "lr_actor = 0.0009 #0.0009\n",
    "lr_critic = 0.001 #0.001\n",
    "optimizer_actor = torch.optim.Adam(actor_model.parameters(), lr=lr_actor)\n",
    "optimizer_critic = torch.optim.Adam(critic_model.parameters(), lr=lr_critic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_fn_actor(pred, delta, discount): \n",
    "    return -1 * discount * delta * torch.sum(torch.log(pred))\n",
    "\n",
    "loss_fn_critic = torch.nn.MSELoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:deeprl]",
   "language": "python",
   "name": "conda-env-deeprl-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
